From eaddf264ce7550654a86a3916d76fb955557810a Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Sun, 27 Dec 2020 12:16:06 +0100 Subject: [PATCH 01/26] Add test and coverage reports --- .gitlab-ci.yml | 4 ++++ Taskfile.yml | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3af8f37..ec52ec8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,10 @@ test: stage: test script: - task cli-cover-report + artifacts: + reports: + junit: out/report.xml + cobertura: out/coverage.xml lint: stage: test diff --git a/Taskfile.yml b/Taskfile.yml index bada179..3e438ce 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -46,8 +46,9 @@ tasks: - generate cmds: - mkdir -p {{ .OUT_DIR }} - - go test -coverprofile={{ .OUT_DIR }}/cov-raw.out -v ./... + - go test -coverprofile={{ .OUT_DIR }}/cov-raw.out -covermode count -v ./... 2>&1 | go-junit-report > {{ .OUT_DIR }}/report.xml - grep -v "generated" {{ .OUT_DIR }}/cov-raw.out > {{ .OUT_DIR }}/cov.out + - gocover-cobertura < {{ .OUT_DIR }}/cov.out > {{ .OUT_DIR }}/coverage.xml - rm -f {{ .OUT_DIR }}/cov-raw.out cli-cover-report: From 1e8139e8458fc9e2f1a9b22fa0efd02e9ead9bea Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Wed, 30 Dec 2020 17:03:01 +0100 Subject: [PATCH 02/26] Add first draft of event stream --- Taskfile.yml | 7 +- api/{ => proto/internal/rpc}/endpoints.proto | 2 +- api/{ => proto/internal/rpc}/handlers.proto | 2 +- api/{ => proto/internal/rpc}/health.proto | 2 +- api/proto/pkg/audit/event_entity.proto | 61 +++++ api/proto/pkg/audit/http_details.proto | 32 +++ go.mod | 2 + go.sum | 51 ++-- pkg/audit/api.go | 28 +++ pkg/audit/event.go | 25 ++ pkg/audit/event_stream.go | 117 +++++++++ pkg/audit/event_stream_test.go | 235 +++++++++++++++++++ pkg/audit/http_details.go | 39 +++ pkg/audit/options.go | 54 +++++ pkg/audit/tls_details.go | 36 +++ pkg/logging/factory.go | 8 + pkg/logging/test_logger.go | 131 +++++++++++ 17 files changed, 808 insertions(+), 24 deletions(-) rename api/{ => proto/internal/rpc}/endpoints.proto (96%) rename api/{ => proto/internal/rpc}/handlers.proto (95%) rename api/{ => proto/internal/rpc}/health.proto (96%) create mode 100644 api/proto/pkg/audit/event_entity.proto create mode 100644 api/proto/pkg/audit/http_details.proto create mode 100644 pkg/audit/api.go create mode 100644 pkg/audit/event.go create mode 100644 pkg/audit/event_stream.go create mode 100644 pkg/audit/event_stream_test.go create mode 100644 pkg/audit/http_details.go create mode 100644 pkg/audit/options.go create mode 100644 pkg/audit/tls_details.go create mode 100644 pkg/logging/test_logger.go diff --git a/Taskfile.yml b/Taskfile.yml index 3e438ce..e58d684 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -5,7 +5,7 @@ vars: INETMOCK_PKG: gitlab.com/inetmock/inetmock/cmd/inetmock IMCTL_PKG: gitlab.com/inetmock/inetmock/cmd/imctl PROTO_FILES: - sh: find ./api/ -type f -name "*.proto" -printf "%f " + sh: find ./api/ -type f -name "*.proto" -printf "%p " env: GOOS: linux @@ -26,7 +26,7 @@ tasks: sources: - "**/*.proto" cmds: - - protoc --proto_path ./api/ --go_out=./internal/rpc --go_opt=paths=source_relative --go-grpc_out=./internal/rpc --go-grpc_opt=paths=source_relative {{ .PROTO_FILES }} + - protoc --proto_path ./api/proto/ --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative {{ .PROTO_FILES }} go-generate: sources: @@ -46,7 +46,8 @@ tasks: - generate cmds: - mkdir -p {{ .OUT_DIR }} - - go test -coverprofile={{ .OUT_DIR }}/cov-raw.out -covermode count -v ./... 2>&1 | go-junit-report > {{ .OUT_DIR }}/report.xml + - go test -coverprofile={{ .OUT_DIR }}/cov-raw.out -covermode count -v ./... 2>&1 | tee {{ .OUT_DIR }}/test_output + - cat {{ .OUT_DIR }}/test_output | go-junit-report -set-exit-code > {{ .OUT_DIR }}/report.xml - grep -v "generated" {{ .OUT_DIR }}/cov-raw.out > {{ .OUT_DIR }}/cov.out - gocover-cobertura < {{ .OUT_DIR }}/cov.out > {{ .OUT_DIR }}/coverage.xml - rm -f {{ .OUT_DIR }}/cov-raw.out diff --git a/api/endpoints.proto b/api/proto/internal/rpc/endpoints.proto similarity index 96% rename from api/endpoints.proto rename to api/proto/internal/rpc/endpoints.proto index 3981ff5..48d850b 100644 --- a/api/endpoints.proto +++ b/api/proto/internal/rpc/endpoints.proto @@ -5,7 +5,7 @@ option java_multiple_files = true; option java_package = "com.github.baez90.inetmock.rpc"; option java_outer_classname = "EndpointsProto"; -package inetmock; +package inetmock.rpc; service Endpoints { rpc GetEndpoints (GetEndpointsRequest) returns (GetEndpointsResponse) { diff --git a/api/handlers.proto b/api/proto/internal/rpc/handlers.proto similarity index 95% rename from api/handlers.proto rename to api/proto/internal/rpc/handlers.proto index 6440a99..44a3ff9 100644 --- a/api/handlers.proto +++ b/api/proto/internal/rpc/handlers.proto @@ -5,7 +5,7 @@ option java_multiple_files = true; option java_package = "com.github.baez90.inetmock.rpc"; option java_outer_classname = "HandlersProto"; -package inetmock; +package inetmock.rpc; service Handlers { rpc GetHandlers (GetHandlersRequest) returns (GetHandlersResponse) { diff --git a/api/health.proto b/api/proto/internal/rpc/health.proto similarity index 96% rename from api/health.proto rename to api/proto/internal/rpc/health.proto index 8c9b6b9..3ff1aee 100644 --- a/api/health.proto +++ b/api/proto/internal/rpc/health.proto @@ -5,7 +5,7 @@ option java_multiple_files = true; option java_package = "com.github.baez90.inetmock.rpc"; option java_outer_classname = "HealthProto"; -package inetmock; +package inetmock.rpc; service Health { rpc GetHealth (HealthRequest) returns (HealthResponse) { diff --git a/api/proto/pkg/audit/event_entity.proto b/api/proto/pkg/audit/event_entity.proto new file mode 100644 index 0000000..2939193 --- /dev/null +++ b/api/proto/pkg/audit/event_entity.proto @@ -0,0 +1,61 @@ +syntax = "proto3"; + +option go_package = "gitlab.com/inetmock/inetmock/pkg/audit"; +option java_multiple_files = true; +option java_package = "com.github.baez90.inetmock.audit"; +option java_outer_classname = "HandlerEventProto"; + +package inetmock.audit; + +import "google/protobuf/any.proto"; +import "google/protobuf/timestamp.proto"; + +enum TransportProtocol { + UNKNOWN_TRANSPORT = 0; + TCP = 1; + UDP = 2; +} + +enum AppProtocol { + UNKNOWN_APPLICATION = 0; + DNS = 1; + HTTP = 2; + HTTP_PROXY = 3; +} + +enum TLSVersion { + SSLv30 = 0; + TLS10 = 1; + TLS11 = 2; + TLS12 = 3; + TLS13 = 4; +} + +message TLSDetailsEntity { + TLSVersion version = 1; + string cipherSuite = 2; + string serverName = 3; +} + +message EventEntity { + int64 id = 1; + google.protobuf.Timestamp timestamp = 2; + TransportProtocol transport = 3; + AppProtocol application = 4; + + oneof sourceIP { + uint32 sourceIPv4 = 5; + uint64 sourceIPv6 = 6; + } + + oneof destinationIP { + uint32 destinationIPv4 = 7; + uint64 destinationIPv6 = 8; + } + + uint32 sourcePort = 9; + uint32 destinationPort = 10; + + TLSDetailsEntity tls = 11; + google.protobuf.Any protocolDetails = 12; +} \ No newline at end of file diff --git a/api/proto/pkg/audit/http_details.proto b/api/proto/pkg/audit/http_details.proto new file mode 100644 index 0000000..a771024 --- /dev/null +++ b/api/proto/pkg/audit/http_details.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +option go_package = "gitlab.com/inetmock/inetmock/pkg/audit"; +option java_multiple_files = true; +option java_package = "com.github.baez90.inetmock.audit"; +option java_outer_classname = "HandlerEventProto"; + +package inetmock.audit; + +enum HTTPMethod { + GET = 0; + HEAD = 1; + POST = 2; + PUT = 3; + DELETE = 4; + CONNECT = 5; + OPTIONS = 6; + TRACE = 7; + PATCH = 8; +} + +message HTTPHeaderValue { + repeated string values = 1; +} + +message HTTPDetailsEntity { + HTTPMethod method = 1; + string host = 2; + string uri = 3; + string proto = 4; + map headers = 5; +} \ No newline at end of file diff --git a/go.mod b/go.mod index e17d974..993562b 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,11 @@ module gitlab.com/inetmock/inetmock go 1.15 require ( + github.com/bwmarrin/snowflake v0.3.0 github.com/golang/mock v1.4.4 github.com/golang/protobuf v1.4.2 github.com/google/uuid v1.1.2 + github.com/imdario/mergo v0.3.11 github.com/miekg/dns v1.1.31 github.com/olekukonko/tablewriter v0.0.4 github.com/prometheus/client_golang v1.7.1 diff --git a/go.sum b/go.sum index fec03c6..d1a0e24 100644 --- a/go.sum +++ b/go.sum @@ -30,13 +30,14 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= +github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -58,12 +59,13 @@ github.com/elazarl/goproxy/ext v0.0.0-20201021153353-00ad82a08272 h1:xOHQWEGsftk github.com/elazarl/goproxy/ext v0.0.0-20201021153353-00ad82a08272/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -83,7 +85,6 @@ github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -92,8 +93,6 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -108,8 +107,6 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -142,6 +139,8 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -154,6 +153,7 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -163,6 +163,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.4 h1:8KGKTcQQGm0Kv7vEbKFErAoAOFyyacLStRtQSeYtvkY= +github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= @@ -182,6 +184,8 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.0 h1:7ks8ZkOP5/ujthUsT07rNv+nkLXCQWKNHuwzOAesEks= +github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -193,9 +197,12 @@ github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FW github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -241,12 +248,18 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.5.1 h1:VHu76Lk0LSP1x254maIu2bplkWpfBWI+B+6fdoZprcg= +github.com/spf13/afero v1.5.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -259,6 +272,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -272,9 +286,13 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -286,6 +304,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -327,8 +346,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -356,13 +373,12 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201223074533-0d417f636930 h1:vRgIt+nup/B/BwIS0g2oC0haq0iqbV3ZA+u6+0TlNCo= golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -420,8 +436,6 @@ google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgX google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d h1:HV9Z9qMhQEsdlvxNFELgQ11RkMzO3CMkjEySjCtuLes= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -429,12 +443,8 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.1 h1:M8spwkmx0pHrPq+uMdl22w5CvJ/Y+oAJTIs9oGoCpOE= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.1/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -444,7 +454,6 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -458,6 +467,8 @@ gopkg.in/elazarl/goproxy.v1 v1.0.0-20180725130230-947c36da3153/go.mod h1:xzjpkye gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -466,6 +477,10 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/audit/api.go b/pkg/audit/api.go new file mode 100644 index 0000000..fa4ef7b --- /dev/null +++ b/pkg/audit/api.go @@ -0,0 +1,28 @@ +//go:generate mockgen -source=$GOFILE -destination=./../../internal/mock/audit/audit.mock.go -package=audit_mock + +package audit + +import ( + "errors" + "io" +) + +var ( + ErrSinkAlreadyRegistered = errors.New("sink with same name already registered") +) + +type Emitter interface { + Emit(ev Event) +} + +type Sink interface { + Name() string + OnSubscribe(evs <-chan Event) +} + +type EventStream interface { + io.Closer + Emitter + RegisterSink(s Sink) error + Sinks() []string +} diff --git a/pkg/audit/event.go b/pkg/audit/event.go new file mode 100644 index 0000000..e625734 --- /dev/null +++ b/pkg/audit/event.go @@ -0,0 +1,25 @@ +package audit + +import ( + "net" + "time" + + "google.golang.org/protobuf/proto" +) + +type EventDetails interface { + ProtoMessage() proto.Message +} + +type Event struct { + ID int64 + Timestamp time.Time + Transport TransportProtocol + Application AppProtocol + SourceIP net.IP + DestinationIP net.IP + SourcePort uint16 + DestinationPort uint16 + ProtocolDetails EventDetails + TLS *TLSDetails +} diff --git a/pkg/audit/event_stream.go b/pkg/audit/event_stream.go new file mode 100644 index 0000000..0019805 --- /dev/null +++ b/pkg/audit/event_stream.go @@ -0,0 +1,117 @@ +package audit + +import ( + "sync" + "time" + + "github.com/bwmarrin/snowflake" + "github.com/imdario/mergo" + "gitlab.com/inetmock/inetmock/pkg/logging" + "go.uber.org/zap" +) + +func init() { + snowflake.Epoch = time.Unix(0, 0).Unix() +} + +type eventStream struct { + logger logging.Logger + buffer chan *Event + sinks map[string]chan Event + lock sync.Locker + idGenerator *snowflake.Node + sinkBufferSize int + sinkConsumptionTimeout time.Duration +} + +func NewEventStream(logger logging.Logger, options ...EventStreamOption) (es EventStream, err error) { + cfg := newEventStreamCfg() + + for _, opt := range options { + opt(&cfg) + } + + var node *snowflake.Node + node, err = snowflake.NewNode(cfg.generatorIndex) + if err != nil { + return + } + + generatorIdx++ + underlying := &eventStream{ + logger: logger, + sinks: make(map[string]chan Event), + buffer: make(chan *Event, cfg.bufferSize), + sinkBufferSize: cfg.sinkBuffersize, + sinkConsumptionTimeout: cfg.sinkConsumptionTimeout, + idGenerator: node, + lock: &sync.Mutex{}, + } + + go underlying.distribute() + + es = underlying + + return +} + +func (e *eventStream) Emit(ev Event) { + defaultEvent := e.newDefaultEvent() + _ = mergo.Merge(defaultEvent, &ev) + select { + case e.buffer <- defaultEvent: + e.logger.Debug("pushed event to distribute loop") + default: + e.logger.Warn("buffer is full") + } +} + +func (e *eventStream) RegisterSink(s Sink) error { + name := s.Name() + + if _, exists := e.sinks[name]; exists { + return ErrSinkAlreadyRegistered + } + + downstream := make(chan Event, e.sinkBufferSize) + s.OnSubscribe(downstream) + e.sinks[name] = downstream + return nil +} + +func (e eventStream) Sinks() (sinks []string) { + for name := range e.sinks { + sinks = append(sinks, name) + } + return +} + +func (e *eventStream) Close() error { + close(e.buffer) + for _, downstream := range e.sinks { + close(downstream) + } + return nil +} + +func (e *eventStream) distribute() { + for ev := range e.buffer { + for name, s := range e.sinks { + e.logger.Debug("notify sink", zap.String("sink", name)) + select { + case s <- *ev: + e.logger.Debug("pushed event to sink channel") + case <-time.After(e.sinkConsumptionTimeout): + e.logger.Warn("sink consummation timed out") + } + } + } +} + +func (e *eventStream) newDefaultEvent() *Event { + id := e.idGenerator.Generate() + return &Event{ + ID: id.Int64(), + Timestamp: time.Now().UTC(), + } +} diff --git a/pkg/audit/event_stream_test.go b/pkg/audit/event_stream_test.go new file mode 100644 index 0000000..adf6c08 --- /dev/null +++ b/pkg/audit/event_stream_test.go @@ -0,0 +1,235 @@ +package audit_test + +import ( + "net" + "sync" + "testing" + "time" + + "gitlab.com/inetmock/inetmock/pkg/audit" + "gitlab.com/inetmock/inetmock/pkg/logging" +) + +var ( + defaultSink = &testSink{ + name: "test defaultSink", + } +) + +type testSink struct { + name string + consumer func(event audit.Event) +} + +func (t *testSink) Name() string { + return t.name +} + +func (t *testSink) OnSubscribe(evs <-chan audit.Event) { + go func() { + for ev := range evs { + if t.consumer != nil { + t.consumer(ev) + } + } + }() +} + +func wgMockSink(t testing.TB, wg *sync.WaitGroup) audit.Sink { + return &testSink{ + name: "WG mock sink", + consumer: func(event audit.Event) { + t.Logf("Got event = %v", event) + wg.Done() + }, + } +} + +func Test_eventStream_RegisterSink(t *testing.T) { + type args struct { + s audit.Sink + } + + tests := []struct { + name string + args args + setup func(e audit.EventStream) + wantErr bool + }{ + { + name: "Register test defaultSink", + args: args{ + s: defaultSink, + }, + wantErr: false, + }, + { + name: "Fail due to already registered defaultSink", + args: args{ + s: defaultSink, + }, + setup: func(e audit.EventStream) { + _ = e.RegisterSink(defaultSink) + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var err error + var e audit.EventStream + if e, err = audit.NewEventStream(logging.CreateTestLogger(t)); err != nil { + t.Errorf("NewEventStream() error = %v", err) + } + + t.Cleanup(func() { + _ = e.Close() + }) + + if tt.setup != nil { + tt.setup(e) + } + + if err := e.RegisterSink(tt.args.s); (err != nil) != tt.wantErr { + t.Errorf("RegisterSink() error = %v, wantErr %v", err, tt.wantErr) + } + + found := false + for _, s := range e.Sinks() { + if found = s == tt.args.s.Name(); found { + break + } + } + if !found { + t.Errorf("expected defaultSink name %s not found in registered sinks %v", tt.args.s.Name(), e.Sinks()) + } + }) + } +} + +func Test_eventStream_Emit(t *testing.T) { + type args struct { + evs []audit.Event + opts []audit.EventStreamOption + } + tests := []struct { + name string + args args + subscribe bool + }{ + { + name: "Expect to get a single event", + subscribe: true, + args: args{ + opts: []audit.EventStreamOption{audit.WithBufferSize(10)}, + evs: []audit.Event{ + { + Transport: audit.TransportProtocol_TCP, + Application: audit.AppProtocol_HTTP, + SourceIP: net.ParseIP("127.0.0.1"), + DestinationIP: net.ParseIP("127.0.0.1"), + SourcePort: 32344, + DestinationPort: 80, + }, + }, + }, + }, + { + name: "Expect to get multiple events", + subscribe: true, + args: args{ + opts: []audit.EventStreamOption{audit.WithBufferSize(10)}, + evs: []audit.Event{ + { + Transport: audit.TransportProtocol_TCP, + Application: audit.AppProtocol_HTTP, + SourceIP: net.ParseIP("127.0.0.1"), + DestinationIP: net.ParseIP("127.0.0.1"), + SourcePort: 32344, + DestinationPort: 80, + }, + { + Transport: audit.TransportProtocol_TCP, + Application: audit.AppProtocol_DNS, + SourceIP: net.ParseIP("::1"), + DestinationIP: net.ParseIP("::1"), + SourcePort: 32344, + DestinationPort: 80, + }, + }, + }, + }, + { + name: "Emit without subscribe sink", + args: args{ + opts: []audit.EventStreamOption{audit.WithBufferSize(0)}, + evs: []audit.Event{ + { + Transport: audit.TransportProtocol_TCP, + Application: audit.AppProtocol_HTTP, + SourceIP: net.ParseIP("127.0.0.1"), + DestinationIP: net.ParseIP("127.0.0.1"), + SourcePort: 32344, + DestinationPort: 80, + }, + }, + }, + subscribe: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var err error + var e audit.EventStream + if e, err = audit.NewEventStream(logging.CreateTestLogger(t), tt.args.opts...); err != nil { + t.Errorf("NewEventStream() error = %v", err) + } + + t.Cleanup(func() { + _ = e.Close() + }) + + emittedWaitGroup := &sync.WaitGroup{} + receivedWaitGroup := &sync.WaitGroup{} + + emittedWaitGroup.Add(len(tt.args.evs)) + + if tt.subscribe { + receivedWaitGroup.Add(len(tt.args.evs)) + if err := e.RegisterSink(wgMockSink(t, receivedWaitGroup)); err != nil { + t.Errorf("RegisterSink() error = %v", err) + } + } + + go func(evs []audit.Event, wg *sync.WaitGroup) { + for _, ev := range evs { + e.Emit(ev) + wg.Done() + } + }(tt.args.evs, emittedWaitGroup) + + select { + case <-waitGroupDone(emittedWaitGroup): + case <-time.After(100 * time.Millisecond): + t.Errorf("not all events emitted in time") + } + + select { + case <-waitGroupDone(receivedWaitGroup): + case <-time.After(5 * time.Second): + t.Errorf("did not get all expected events in time") + } + }) + } +} + +func waitGroupDone(wg *sync.WaitGroup) <-chan struct{} { + done := make(chan struct{}) + + go func(wg *sync.WaitGroup) { + wg.Wait() + close(done) + }(wg) + + return done +} diff --git a/pkg/audit/http_details.go b/pkg/audit/http_details.go new file mode 100644 index 0000000..dc5b096 --- /dev/null +++ b/pkg/audit/http_details.go @@ -0,0 +1,39 @@ +package audit + +import ( + "net/http" + "strings" + + "google.golang.org/protobuf/proto" +) + +type HTTPDetails struct { + Method string + Host string + URI string + Proto string + Headers http.Header +} + +func (d HTTPDetails) ProtoMessage() proto.Message { + var method = HTTPMethod_GET + if methodValue, known := HTTPMethod_value[strings.ToUpper(d.Method)]; known { + method = HTTPMethod(methodValue) + } + + headers := make(map[string]*HTTPHeaderValue) + + for k, v := range d.Headers { + headers[k] = &HTTPHeaderValue{ + Values: v, + } + } + + return &HTTPDetailsEntity{ + Method: method, + Host: d.Host, + Uri: d.URI, + Proto: d.Proto, + Headers: headers, + } +} diff --git a/pkg/audit/options.go b/pkg/audit/options.go new file mode 100644 index 0000000..24ee746 --- /dev/null +++ b/pkg/audit/options.go @@ -0,0 +1,54 @@ +package audit + +import "time" + +const ( + defaultEventStreamBufferSize = 100 + defaultSinkBufferSize = 0 + defaultSinkConsumptionTimeout = 50 * time.Millisecond +) + +var ( + generatorIdx int64 = 1 + WithBufferSize = func(bufferSize int) EventStreamOption { + return func(cfg *eventStreamCfg) { + cfg.bufferSize = bufferSize + } + } + WithGeneratorIndex = func(generatorIndex int64) EventStreamOption { + return func(cfg *eventStreamCfg) { + cfg.generatorIndex = generatorIndex + } + } + WithSinkBufferSize = func(bufferSize int) EventStreamOption { + return func(cfg *eventStreamCfg) { + cfg.sinkBuffersize = bufferSize + } + } + WithSinkConsumptionTimeout = func(timeout time.Duration) EventStreamOption { + return func(cfg *eventStreamCfg) { + cfg.sinkConsumptionTimeout = timeout + } + } +) + +type EventStreamOption func(cfg *eventStreamCfg) + +type eventStreamCfg struct { + bufferSize int + sinkBuffersize int + generatorIndex int64 + sinkConsumptionTimeout time.Duration +} + +func newEventStreamCfg() eventStreamCfg { + cfg := eventStreamCfg{ + generatorIndex: generatorIdx, + sinkBuffersize: defaultSinkBufferSize, + bufferSize: defaultEventStreamBufferSize, + sinkConsumptionTimeout: defaultSinkConsumptionTimeout, + } + generatorIdx++ + + return cfg +} diff --git a/pkg/audit/tls_details.go b/pkg/audit/tls_details.go new file mode 100644 index 0000000..fc9ae75 --- /dev/null +++ b/pkg/audit/tls_details.go @@ -0,0 +1,36 @@ +package audit + +import ( + "crypto/tls" + + "google.golang.org/protobuf/proto" +) + +type TLSDetails struct { + Version uint16 + CipherSuite uint16 + ServerName string +} + +func (d TLSDetails) ProtoMessage() proto.Message { + var version TLSVersion + + switch d.Version { + case tls.VersionTLS10: + version = TLSVersion_TLS10 + case tls.VersionTLS11: + version = TLSVersion_TLS11 + case tls.VersionTLS12: + version = TLSVersion_TLS12 + case tls.VersionTLS13: + version = TLSVersion_TLS13 + default: + version = TLSVersion_SSLv30 + } + + return &TLSDetailsEntity{ + Version: version, + CipherSuite: tls.CipherSuiteName(d.CipherSuite), + ServerName: d.ServerName, + } +} diff --git a/pkg/logging/factory.go b/pkg/logging/factory.go index 727d55b..961ee55 100644 --- a/pkg/logging/factory.go +++ b/pkg/logging/factory.go @@ -2,6 +2,7 @@ package logging import ( "strings" + "testing" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -48,6 +49,13 @@ func CreateLogger() (Logger, error) { } } +func CreateTestLogger(tb testing.TB) Logger { + return &testLogger{ + tb: tb, + encoder: zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()), + } +} + func MustCreateLogger() Logger { if logger, err := CreateLogger(); err != nil { panic(err) diff --git a/pkg/logging/test_logger.go b/pkg/logging/test_logger.go new file mode 100644 index 0000000..3ae9a93 --- /dev/null +++ b/pkg/logging/test_logger.go @@ -0,0 +1,131 @@ +package logging + +import ( + "runtime" + "runtime/debug" + "testing" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +type testLogger struct { + name string + fields []zap.Field + tb testing.TB + encoder zapcore.Encoder +} + +func (t testLogger) Named(s string) Logger { + return testLogger{ + name: s, + tb: t.tb, + fields: t.fields, + } +} + +func (t testLogger) With(fields ...zap.Field) Logger { + return &testLogger{ + name: t.name, + fields: append(t.fields, fields...), + tb: t.tb, + } +} + +func (t testLogger) Debug(msg string, fields ...zap.Field) { + t.tb.Helper() + buf, err := t.encoder.EncodeEntry(zapcore.Entry{ + Level: zapcore.DebugLevel, + Time: time.Now(), + LoggerName: t.name, + Message: msg, + Caller: zapcore.NewEntryCaller(runtime.Caller(2)), + }, append(t.fields, fields...)) + + if err == nil { + t.tb.Log(buf.String()) + } +} + +func (t testLogger) Info(msg string, fields ...zap.Field) { + t.tb.Helper() + buf, err := t.encoder.EncodeEntry(zapcore.Entry{ + Level: zapcore.InfoLevel, + Time: time.Now(), + LoggerName: t.name, + Message: msg, + Caller: zapcore.NewEntryCaller(runtime.Caller(2)), + }, append(t.fields, fields...)) + + if err == nil { + t.tb.Log(buf.String()) + } +} + +func (t testLogger) Warn(msg string, fields ...zap.Field) { + t.tb.Helper() + buf, err := t.encoder.EncodeEntry(zapcore.Entry{ + Level: zapcore.WarnLevel, + Time: time.Now(), + LoggerName: t.name, + Message: msg, + Caller: zapcore.NewEntryCaller(runtime.Caller(2)), + }, append(t.fields, fields...)) + + if err == nil { + t.tb.Log(buf.String()) + } +} + +func (t testLogger) Error(msg string, fields ...zap.Field) { + t.tb.Helper() + buf, err := t.encoder.EncodeEntry(zapcore.Entry{ + Level: zapcore.ErrorLevel, + Time: time.Now(), + LoggerName: t.name, + Message: msg, + Caller: zapcore.NewEntryCaller(runtime.Caller(2)), + Stack: string(debug.Stack()), + }, append(t.fields, fields...)) + + if err == nil { + t.tb.Log(buf.String()) + } +} + +func (t testLogger) Panic(msg string, fields ...zap.Field) { + t.tb.Helper() + buf, err := t.encoder.EncodeEntry(zapcore.Entry{ + Level: zapcore.PanicLevel, + Time: time.Now(), + LoggerName: t.name, + Message: msg, + Caller: zapcore.NewEntryCaller(runtime.Caller(2)), + Stack: string(debug.Stack()), + }, append(t.fields, fields...)) + + if err == nil { + t.tb.Error(buf.String()) + } +} + +func (t testLogger) Fatal(msg string, fields ...zap.Field) { + t.tb.Helper() + buf, err := t.encoder.EncodeEntry(zapcore.Entry{ + Level: zapcore.FatalLevel, + Time: time.Now(), + LoggerName: t.name, + Message: msg, + Caller: zapcore.NewEntryCaller(runtime.Caller(2)), + Stack: string(debug.Stack()), + }, append(t.fields, fields...)) + + if err == nil { + t.tb.Error(buf.String()) + } +} + +func (t testLogger) Sync() error { + return nil +} From 63607dfd6755639493e4ffe837015fcaefce4dc6 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Sat, 2 Jan 2021 17:24:06 +0100 Subject: [PATCH 03/26] Implement log and writer sinks Add reader --- internal/endpoints/endpoint_manager.go | 4 +- pkg/audit/event.go | 102 ++++++++++++++++++++- pkg/audit/event_stream.go | 14 +-- pkg/audit/event_stream_test.go | 120 ++++++++++++++----------- pkg/audit/http_details.go | 9 +- pkg/audit/log_sink.go | 53 +++++++++++ pkg/audit/log_sink_test.go | 110 +++++++++++++++++++++++ pkg/audit/reader.go | 87 ++++++++++++++++++ pkg/audit/reader_test.go | 97 ++++++++++++++++++++ pkg/audit/tls_details.go | 52 +++++++---- pkg/audit/writer.go | 85 ++++++++++++++++++ pkg/audit/writer_sink.go | 47 ++++++++++ pkg/audit/writer_sink_test.go | 72 +++++++++++++++ pkg/audit/writer_test.go | 117 ++++++++++++++++++++++++ plugins/tls_interceptor/register.go | 2 +- 15 files changed, 883 insertions(+), 88 deletions(-) create mode 100644 pkg/audit/log_sink.go create mode 100644 pkg/audit/log_sink_test.go create mode 100644 pkg/audit/reader.go create mode 100644 pkg/audit/reader_test.go create mode 100644 pkg/audit/writer.go create mode 100644 pkg/audit/writer_sink.go create mode 100644 pkg/audit/writer_sink_test.go create mode 100644 pkg/audit/writer_test.go diff --git a/internal/endpoints/endpoint_manager.go b/internal/endpoints/endpoint_manager.go index afc13e2..0470f8d 100644 --- a/internal/endpoints/endpoint_manager.go +++ b/internal/endpoints/endpoint_manager.go @@ -99,7 +99,7 @@ func (e *endpointManager) StartEndpoints() { } func (e *endpointManager) ShutdownEndpoints() { - var waitGroup sync.WaitGroup + waitGroup := new(sync.WaitGroup) waitGroup.Add(len(e.properlyStartedEndpoints)) parentCtx, _ := context.WithTimeout(context.Background(), shutdownTimeout) @@ -112,7 +112,7 @@ func (e *endpointManager) ShutdownEndpoints() { zap.String("endpoint", endpoint.Name()), ) endpointLogger.Info("Triggering shutdown of endpoint") - go shutdownEndpoint(ctx, endpoint, endpointLogger, &waitGroup) + go shutdownEndpoint(ctx, endpoint, endpointLogger, waitGroup) } waitGroup.Wait() diff --git a/pkg/audit/event.go b/pkg/audit/event.go index e625734..84fb7d1 100644 --- a/pkg/audit/event.go +++ b/pkg/audit/event.go @@ -1,14 +1,18 @@ package audit import ( + "encoding/binary" + "math/big" "net" "time" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/timestamppb" ) -type EventDetails interface { - ProtoMessage() proto.Message +type Details interface { + MarshalToWireFormat() (*anypb.Any, error) } type Event struct { @@ -20,6 +24,98 @@ type Event struct { DestinationIP net.IP SourcePort uint16 DestinationPort uint16 - ProtocolDetails EventDetails + ProtocolDetails *anypb.Any TLS *TLSDetails } + +func (e *Event) ProtoMessage() proto.Message { + var sourceIP isEventEntity_SourceIP + if ipv4 := e.SourceIP.To4(); ipv4 != nil { + if len(ipv4) == 16 { + sourceIP = &EventEntity_SourceIPv4{SourceIPv4: binary.BigEndian.Uint32(ipv4[12:16])} + } else { + sourceIP = &EventEntity_SourceIPv4{SourceIPv4: binary.BigEndian.Uint32(ipv4)} + } + } else { + ipv6 := big.NewInt(0) + ipv6.SetBytes(e.SourceIP) + sourceIP = &EventEntity_SourceIPv6{SourceIPv6: ipv6.Uint64()} + } + + var destinationIP isEventEntity_DestinationIP + if ipv4 := e.DestinationIP.To4(); ipv4 != nil { + if len(ipv4) == 16 { + destinationIP = &EventEntity_DestinationIPv4{DestinationIPv4: binary.BigEndian.Uint32(ipv4[12:16])} + } else { + destinationIP = &EventEntity_DestinationIPv4{DestinationIPv4: binary.BigEndian.Uint32(ipv4)} + } + } else { + ipv6 := big.NewInt(0) + ipv6.SetBytes(e.SourceIP) + destinationIP = &EventEntity_DestinationIPv6{DestinationIPv6: ipv6.Uint64()} + } + + var tlsDetails *TLSDetailsEntity = nil + if e.TLS != nil { + tlsDetails = e.TLS.ProtoMessage() + } + + return &EventEntity{ + Id: e.ID, + Timestamp: timestamppb.New(e.Timestamp), + Transport: e.Transport, + Application: e.Application, + SourceIP: sourceIP, + DestinationIP: destinationIP, + SourcePort: uint32(e.SourcePort), + DestinationPort: uint32(e.DestinationPort), + Tls: tlsDetails, + ProtocolDetails: e.ProtocolDetails, + } +} + +func (e *Event) ApplyDefaults(id int64) { + e.ID = id + emptyTime := time.Time{} + if e.Timestamp == emptyTime { + e.Timestamp = time.Now().UTC() + } +} + +func NewEventFromProto(msg *EventEntity) (ev Event) { + var sourceIP net.IP + switch ip := msg.GetSourceIP().(type) { + case *EventEntity_SourceIPv4: + buf := make([]byte, 4) + binary.BigEndian.PutUint32(buf, ip.SourceIPv4) + sourceIP = buf + sourceIP = sourceIP.To4() + case *EventEntity_SourceIPv6: + sourceIP = big.NewInt(int64(ip.SourceIPv6)).Bytes() + } + + var destinationIP net.IP + switch ip := msg.GetDestinationIP().(type) { + case *EventEntity_DestinationIPv4: + buf := make([]byte, 4) + binary.BigEndian.PutUint32(buf, ip.DestinationIPv4) + destinationIP = buf + destinationIP = destinationIP.To4() + case *EventEntity_DestinationIPv6: + destinationIP = big.NewInt(int64(ip.DestinationIPv6)).Bytes() + } + + ev = Event{ + ID: msg.GetId(), + Timestamp: msg.GetTimestamp().AsTime(), + Transport: msg.GetTransport(), + Application: msg.GetApplication(), + SourceIP: sourceIP, + DestinationIP: destinationIP, + SourcePort: uint16(msg.GetSourcePort()), + DestinationPort: uint16(msg.GetDestinationPort()), + ProtocolDetails: msg.GetProtocolDetails(), + TLS: NewTLSDetailsFromProto(msg.GetTls()), + } + return +} diff --git a/pkg/audit/event_stream.go b/pkg/audit/event_stream.go index 0019805..bc22331 100644 --- a/pkg/audit/event_stream.go +++ b/pkg/audit/event_stream.go @@ -5,7 +5,6 @@ import ( "time" "github.com/bwmarrin/snowflake" - "github.com/imdario/mergo" "gitlab.com/inetmock/inetmock/pkg/logging" "go.uber.org/zap" ) @@ -56,10 +55,9 @@ func NewEventStream(logger logging.Logger, options ...EventStreamOption) (es Eve } func (e *eventStream) Emit(ev Event) { - defaultEvent := e.newDefaultEvent() - _ = mergo.Merge(defaultEvent, &ev) + ev.ApplyDefaults(e.idGenerator.Generate().Int64()) select { - case e.buffer <- defaultEvent: + case e.buffer <- &ev: e.logger.Debug("pushed event to distribute loop") default: e.logger.Warn("buffer is full") @@ -107,11 +105,3 @@ func (e *eventStream) distribute() { } } } - -func (e *eventStream) newDefaultEvent() *Event { - id := e.idGenerator.Generate() - return &Event{ - ID: id.Int64(), - Timestamp: time.Now().UTC(), - } -} diff --git a/pkg/audit/event_stream_test.go b/pkg/audit/event_stream_test.go index adf6c08..17732e3 100644 --- a/pkg/audit/event_stream_test.go +++ b/pkg/audit/event_stream_test.go @@ -1,19 +1,54 @@ package audit_test import ( + "crypto/tls" "net" + "net/http" "sync" "testing" "time" "gitlab.com/inetmock/inetmock/pkg/audit" "gitlab.com/inetmock/inetmock/pkg/logging" + "google.golang.org/protobuf/types/known/anypb" ) var ( defaultSink = &testSink{ name: "test defaultSink", } + testEvents = []audit.Event{ + { + Transport: audit.TransportProtocol_TCP, + Application: audit.AppProtocol_HTTP, + SourceIP: net.ParseIP("127.0.0.1"), + DestinationIP: net.ParseIP("127.0.0.1"), + SourcePort: 32344, + DestinationPort: 80, + TLS: &audit.TLSDetails{ + Version: tls.VersionTLS13, + CipherSuite: tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + ServerName: "localhost", + }, + ProtocolDetails: mustMarshalToWireformat(audit.HTTPDetails{ + Method: "GET", + Host: "localhost", + URI: "http://localhost/asdf", + Proto: "HTTP 1.1", + Headers: http.Header{ + "Accept": []string{"application/json"}, + }, + }), + }, + { + Transport: audit.TransportProtocol_TCP, + Application: audit.AppProtocol_DNS, + SourceIP: net.ParseIP("::1"), + DestinationIP: net.ParseIP("::1"), + SourcePort: 32344, + DestinationPort: 80, + }, + } ) type testSink struct { @@ -45,17 +80,25 @@ func wgMockSink(t testing.TB, wg *sync.WaitGroup) audit.Sink { } } +func mustMarshalToWireformat(d audit.Details) *anypb.Any { + any, err := d.MarshalToWireFormat() + if err != nil { + panic(err) + } + return any +} + func Test_eventStream_RegisterSink(t *testing.T) { type args struct { s audit.Sink } - - tests := []struct { + type testCase struct { name string args args setup func(e audit.EventStream) wantErr bool - }{ + } + tests := []testCase{ { name: "Register test defaultSink", args: args{ @@ -74,8 +117,8 @@ func Test_eventStream_RegisterSink(t *testing.T) { wantErr: true, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + scenario := func(tt testCase) func(t *testing.T) { + return func(t *testing.T) { var err error var e audit.EventStream if e, err = audit.NewEventStream(logging.CreateTestLogger(t)); err != nil { @@ -103,7 +146,11 @@ func Test_eventStream_RegisterSink(t *testing.T) { if !found { t.Errorf("expected defaultSink name %s not found in registered sinks %v", tt.args.s.Name(), e.Sinks()) } - }) + } + } + + for _, tt := range tests { + t.Run(tt.name, scenario(tt)) } } @@ -112,26 +159,18 @@ func Test_eventStream_Emit(t *testing.T) { evs []audit.Event opts []audit.EventStreamOption } - tests := []struct { + type testCase struct { name string args args subscribe bool - }{ + } + tests := []testCase{ { name: "Expect to get a single event", subscribe: true, args: args{ opts: []audit.EventStreamOption{audit.WithBufferSize(10)}, - evs: []audit.Event{ - { - Transport: audit.TransportProtocol_TCP, - Application: audit.AppProtocol_HTTP, - SourceIP: net.ParseIP("127.0.0.1"), - DestinationIP: net.ParseIP("127.0.0.1"), - SourcePort: 32344, - DestinationPort: 80, - }, - }, + evs: testEvents[:1], }, }, { @@ -139,46 +178,21 @@ func Test_eventStream_Emit(t *testing.T) { subscribe: true, args: args{ opts: []audit.EventStreamOption{audit.WithBufferSize(10)}, - evs: []audit.Event{ - { - Transport: audit.TransportProtocol_TCP, - Application: audit.AppProtocol_HTTP, - SourceIP: net.ParseIP("127.0.0.1"), - DestinationIP: net.ParseIP("127.0.0.1"), - SourcePort: 32344, - DestinationPort: 80, - }, - { - Transport: audit.TransportProtocol_TCP, - Application: audit.AppProtocol_DNS, - SourceIP: net.ParseIP("::1"), - DestinationIP: net.ParseIP("::1"), - SourcePort: 32344, - DestinationPort: 80, - }, - }, + evs: testEvents, }, }, { name: "Emit without subscribe sink", args: args{ opts: []audit.EventStreamOption{audit.WithBufferSize(0)}, - evs: []audit.Event{ - { - Transport: audit.TransportProtocol_TCP, - Application: audit.AppProtocol_HTTP, - SourceIP: net.ParseIP("127.0.0.1"), - DestinationIP: net.ParseIP("127.0.0.1"), - SourcePort: 32344, - DestinationPort: 80, - }, - }, + evs: testEvents[:1], }, subscribe: false, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + + scenario := func(tt testCase) func(t *testing.T) { + return func(t *testing.T) { var err error var e audit.EventStream if e, err = audit.NewEventStream(logging.CreateTestLogger(t), tt.args.opts...); err != nil { @@ -189,8 +203,8 @@ func Test_eventStream_Emit(t *testing.T) { _ = e.Close() }) - emittedWaitGroup := &sync.WaitGroup{} - receivedWaitGroup := &sync.WaitGroup{} + emittedWaitGroup := new(sync.WaitGroup) + receivedWaitGroup := new(sync.WaitGroup) emittedWaitGroup.Add(len(tt.args.evs)) @@ -219,7 +233,11 @@ func Test_eventStream_Emit(t *testing.T) { case <-time.After(5 * time.Second): t.Errorf("did not get all expected events in time") } - }) + } + } + + for _, tt := range tests { + t.Run(tt.name, scenario(tt)) } } diff --git a/pkg/audit/http_details.go b/pkg/audit/http_details.go index dc5b096..b755a45 100644 --- a/pkg/audit/http_details.go +++ b/pkg/audit/http_details.go @@ -4,7 +4,7 @@ import ( "net/http" "strings" - "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" ) type HTTPDetails struct { @@ -15,7 +15,7 @@ type HTTPDetails struct { Headers http.Header } -func (d HTTPDetails) ProtoMessage() proto.Message { +func (d HTTPDetails) MarshalToWireFormat() (any *anypb.Any, err error) { var method = HTTPMethod_GET if methodValue, known := HTTPMethod_value[strings.ToUpper(d.Method)]; known { method = HTTPMethod(methodValue) @@ -29,11 +29,14 @@ func (d HTTPDetails) ProtoMessage() proto.Message { } } - return &HTTPDetailsEntity{ + protoDetails := &HTTPDetailsEntity{ Method: method, Host: d.Host, Uri: d.URI, Proto: d.Proto, Headers: headers, } + + any, err = anypb.New(protoDetails) + return } diff --git a/pkg/audit/log_sink.go b/pkg/audit/log_sink.go new file mode 100644 index 0000000..a6e07f1 --- /dev/null +++ b/pkg/audit/log_sink.go @@ -0,0 +1,53 @@ +package audit + +import ( + "crypto/tls" + + "gitlab.com/inetmock/inetmock/pkg/logging" + "go.uber.org/zap" +) + +const ( + logSinkName = "logging" +) + +func NewLogSink(logger logging.Logger) Sink { + return &logSink{ + logger: logger, + } +} + +type logSink struct { + logger logging.Logger +} + +func (logSink) Name() string { + return logSinkName +} + +func (l logSink) OnSubscribe(evs <-chan Event) { + go func(logger logging.Logger, evs <-chan Event) { + for ev := range evs { + eventLogger := logger + + if ev.TLS != nil { + eventLogger = eventLogger.With( + zap.String("tls_server_name", ev.TLS.ServerName), + zap.String("tls_cipher_suite", tls.CipherSuiteName(ev.TLS.CipherSuite)), + ) + } + + eventLogger.Info( + "handled request", + zap.Time("timestamp", ev.Timestamp), + zap.String("application", ev.Application.String()), + zap.String("transport", ev.Transport.String()), + zap.String("source_ip", ev.SourceIP.String()), + zap.Uint16("source_port", ev.SourcePort), + zap.String("destination_ip", ev.DestinationIP.String()), + zap.Uint16("destination_port", ev.DestinationPort), + zap.Any("details", ev.ProtocolDetails), + ) + } + }(l.logger, evs) +} diff --git a/pkg/audit/log_sink_test.go b/pkg/audit/log_sink_test.go new file mode 100644 index 0000000..085dc25 --- /dev/null +++ b/pkg/audit/log_sink_test.go @@ -0,0 +1,110 @@ +package audit_test + +import ( + "sync" + "testing" + "time" + + "github.com/golang/mock/gomock" + logging_mock "gitlab.com/inetmock/inetmock/internal/mock/logging" + "gitlab.com/inetmock/inetmock/pkg/audit" + "gitlab.com/inetmock/inetmock/pkg/logging" + "go.uber.org/zap" +) + +//nolint:dupl +func Test_logSink_OnSubscribe(t *testing.T) { + type fields struct { + loggerSetup func(t *testing.T, wg *sync.WaitGroup) logging.Logger + } + type testCase struct { + name string + fields fields + events []audit.Event + } + tests := []testCase{ + { + name: "Get a single log line", + fields: fields{ + loggerSetup: func(t *testing.T, wg *sync.WaitGroup) logging.Logger { + ctrl := gomock.NewController(t) + t.Cleanup(ctrl.Finish) + loggerMock := logging_mock.NewMockLogger(ctrl) + + loggerMock. + EXPECT(). + With(gomock.Any()). + Return(loggerMock) + + loggerMock. + EXPECT(). + Info("handled request", gomock.Any()). + Do(func(_ string, _ ...zap.Field) { + wg.Done() + }). + Times(1) + + return loggerMock + }, + }, + events: testEvents[:1], + }, + { + name: "Get multiple events", + fields: fields{ + loggerSetup: func(t *testing.T, wg *sync.WaitGroup) logging.Logger { + ctrl := gomock.NewController(t) + t.Cleanup(ctrl.Finish) + loggerMock := logging_mock.NewMockLogger(ctrl) + + loggerMock. + EXPECT(). + With(gomock.Any()). + Return(loggerMock) + + loggerMock. + EXPECT(). + Info("handled request", gomock.Any()). + Do(func(_ string, _ ...zap.Field) { + wg.Done() + }). + Times(2) + + return loggerMock + }, + }, + events: testEvents, + }, + } + scenario := func(tt testCase) func(t *testing.T) { + return func(t *testing.T) { + wg := new(sync.WaitGroup) + wg.Add(len(tt.events)) + + logSink := audit.NewLogSink(tt.fields.loggerSetup(t, wg)) + var evs audit.EventStream + var err error + if evs, err = audit.NewEventStream(logging.CreateTestLogger(t)); err != nil { + t.Errorf("NewEventStream() error = %v", err) + } + + if err = evs.RegisterSink(logSink); err != nil { + t.Errorf("RegisterSink() error = %v", err) + } + + for _, ev := range tt.events { + evs.Emit(ev) + } + + select { + case <-time.After(100 * time.Millisecond): + t.Errorf("not all events recorded in time") + case <-waitGroupDone(wg): + } + } + } + + for _, tt := range tests { + t.Run(tt.name, scenario(tt)) + } +} diff --git a/pkg/audit/reader.go b/pkg/audit/reader.go new file mode 100644 index 0000000..d679565 --- /dev/null +++ b/pkg/audit/reader.go @@ -0,0 +1,87 @@ +package audit + +import ( + "encoding/binary" + "io" + "sync" + + "google.golang.org/protobuf/proto" +) + +const ( + lengthBufferSize = 4 + defaultPayloadBufferSize = 4096 +) + +type Reader interface { + Read() (Event, error) +} + +type EventReaderOption func(reader *eventReader) + +var ( + WithReaderByteOrder = func(order binary.ByteOrder) EventReaderOption { + return func(reader *eventReader) { + reader.byteOrder = order + } + } +) + +func NewEventReader(source io.Reader, opts ...EventReaderOption) Reader { + reader := &eventReader{ + source: source, + byteOrder: defaultOrder, + lengthPool: &sync.Pool{ + New: func() interface{} { + return make([]byte, lengthBufferSize) + }, + }, + payloadPool: &sync.Pool{ + New: func() interface{} { + return make([]byte, defaultPayloadBufferSize) + }, + }, + } + + for _, opt := range opts { + opt(reader) + } + + return reader +} + +type eventReader struct { + lengthPool *sync.Pool + payloadPool *sync.Pool + byteOrder binary.ByteOrder + source io.Reader +} + +func (e eventReader) Read() (ev Event, err error) { + lengthBuf := e.lengthPool.Get().([]byte) + if _, err = e.source.Read(lengthBuf); err != nil { + return + } + + length := e.byteOrder.Uint32(lengthBuf) + var msgBuf []byte + if length <= defaultPayloadBufferSize { + msgBuf = e.payloadPool.Get().([]byte)[:length] + } else { + msgBuf = make([]byte, length) + } + + if _, err = e.source.Read(msgBuf); err != nil { + return + } + + protoEv := new(EventEntity) + + if err = proto.Unmarshal(msgBuf, protoEv); err != nil { + return + } + + ev = NewEventFromProto(protoEv) + + return +} diff --git a/pkg/audit/reader_test.go b/pkg/audit/reader_test.go new file mode 100644 index 0000000..3335e79 --- /dev/null +++ b/pkg/audit/reader_test.go @@ -0,0 +1,97 @@ +package audit_test + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "io" + "testing" + + "gitlab.com/inetmock/inetmock/pkg/audit" + "google.golang.org/protobuf/proto" +) + +var ( + //nolint:lll + httpPayloadBytesLittleEndian = `dd000000120b088092b8c398feffffff011801200248d8fc0150505a3308041224544c535f45434448455f45434453415f574954485f4145535f3235365f4342435f5348411a096c6f63616c686f73746282010a34747970652e676f6f676c65617069732e636f6d2f696e65746d6f636b2e61756469742e4854545044657461696c73456e74697479124a12096c6f63616c686f73741a15687474703a2f2f6c6f63616c686f73742f6173646622084854545020312e312a1c0a0641636365707412120a106170706c69636174696f6e2f6a736f6e28818080f80738818080f807` + //nolint:lll + httpPayloadBytesBigEndian = `000000dd120b088092b8c398feffffff011801200248d8fc0150505a3308041224544c535f45434448455f45434453415f574954485f4145535f3235365f4342435f5348411a096c6f63616c686f73746282010a34747970652e676f6f676c65617069732e636f6d2f696e65746d6f636b2e61756469742e4854545044657461696c73456e74697479124a12096c6f63616c686f73741a15687474703a2f2f6c6f63616c686f73742f6173646622084854545020312e312a1c0a0641636365707412120a106170706c69636174696f6e2f6a736f6e28818080f80738818080f807` + dnsPayloadBytesLittleEndian = `1b000000120b088092b8c398feffffff011801200148d8fc01505030014001` + dnsPayloadBytesBigEndian = `0000001b120b088092b8c398feffffff011801200148d8fc01505030014001` +) + +func mustDecodeHex(hexBytes string) io.Reader { + b, err := hex.DecodeString(hexBytes) + if err != nil { + panic(err) + } + return bytes.NewReader(b) +} + +func Test_eventReader_Read(t *testing.T) { + type fields struct { + source io.Reader + order binary.ByteOrder + } + type testCase struct { + name string + fields fields + wantEv audit.Event + wantErr bool + } + tests := []testCase{ + { + name: "Read HTTP payload - little endian", + fields: fields{ + source: mustDecodeHex(httpPayloadBytesLittleEndian), + order: binary.LittleEndian, + }, + wantEv: testEvents[0], + wantErr: false, + }, + { + name: "Read HTTP payload - big endian", + fields: fields{ + source: mustDecodeHex(httpPayloadBytesBigEndian), + order: binary.BigEndian, + }, + wantEv: testEvents[0], + wantErr: false, + }, + { + name: "Read DNS payload - little endian", + fields: fields{ + source: mustDecodeHex(dnsPayloadBytesLittleEndian), + order: binary.LittleEndian, + }, + wantEv: testEvents[1], + wantErr: false, + }, + { + name: "Read DNS payload - big endian", + fields: fields{ + source: mustDecodeHex(dnsPayloadBytesBigEndian), + order: binary.BigEndian, + }, + wantEv: testEvents[1], + wantErr: false, + }, + } + scenario := func(tt testCase) func(t *testing.T) { + return func(t *testing.T) { + e := audit.NewEventReader(tt.fields.source, audit.WithReaderByteOrder(tt.fields.order)) + gotEv, err := e.Read() + if (err != nil) != tt.wantErr { + t.Errorf("Read() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if err == nil && !proto.Equal(gotEv.ProtoMessage(), tt.wantEv.ProtoMessage()) { + t.Errorf("Read() gotEv = %v, want %v", gotEv, tt.wantEv) + } + } + } + for _, tt := range tests { + t.Run(tt.name, scenario(tt)) + } +} diff --git a/pkg/audit/tls_details.go b/pkg/audit/tls_details.go index fc9ae75..a984007 100644 --- a/pkg/audit/tls_details.go +++ b/pkg/audit/tls_details.go @@ -2,8 +2,31 @@ package audit import ( "crypto/tls" +) - "google.golang.org/protobuf/proto" +var ( + tlsToEntity = map[uint16]TLSVersion{ + tls.VersionSSL30: TLSVersion_SSLv30, + tls.VersionTLS10: TLSVersion_TLS10, + tls.VersionTLS11: TLSVersion_TLS11, + tls.VersionTLS12: TLSVersion_TLS12, + tls.VersionTLS13: TLSVersion_TLS13, + } + entityToTls = map[TLSVersion]uint16{ + TLSVersion_SSLv30: tls.VersionSSL30, + TLSVersion_TLS10: tls.VersionTLS10, + TLSVersion_TLS11: tls.VersionTLS11, + TLSVersion_TLS12: tls.VersionTLS12, + TLSVersion_TLS13: tls.VersionTLS13, + } + cipherSuiteIDLookup = func(name string) uint16 { + for _, cs := range tls.CipherSuites() { + if cs.Name == name { + return cs.ID + } + } + return 0 + } ) type TLSDetails struct { @@ -12,24 +35,21 @@ type TLSDetails struct { ServerName string } -func (d TLSDetails) ProtoMessage() proto.Message { - var version TLSVersion - - switch d.Version { - case tls.VersionTLS10: - version = TLSVersion_TLS10 - case tls.VersionTLS11: - version = TLSVersion_TLS11 - case tls.VersionTLS12: - version = TLSVersion_TLS12 - case tls.VersionTLS13: - version = TLSVersion_TLS13 - default: - version = TLSVersion_SSLv30 +func NewTLSDetailsFromProto(entity *TLSDetailsEntity) *TLSDetails { + if entity == nil { + return nil } + return &TLSDetails{ + Version: entityToTls[entity.GetVersion()], + CipherSuite: cipherSuiteIDLookup(entity.GetCipherSuite()), + ServerName: entity.GetServerName(), + } +} + +func (d TLSDetails) ProtoMessage() *TLSDetailsEntity { return &TLSDetailsEntity{ - Version: version, + Version: tlsToEntity[d.Version], CipherSuite: tls.CipherSuiteName(d.CipherSuite), ServerName: d.ServerName, } diff --git a/pkg/audit/writer.go b/pkg/audit/writer.go new file mode 100644 index 0000000..5a318b4 --- /dev/null +++ b/pkg/audit/writer.go @@ -0,0 +1,85 @@ +//go:generate mockgen -source=$GOFILE -destination=./../../internal/mock/audit/writer.mock.go -package=audit_mock + +package audit + +import ( + "encoding/binary" + "errors" + "io" + "sync" + + "google.golang.org/protobuf/proto" +) + +var ( + WithWriterByteOrder = func(order binary.ByteOrder) func(writer *eventWriter) { + return func(writer *eventWriter) { + writer.byteOrder = order + } + } + defaultOrder = binary.BigEndian + ErrValueMostNotBeNil = errors.New("event value must not be nil") +) + +type Writer interface { + io.Closer + Write(ev *Event) error +} + +type EventWriterOption func(writer *eventWriter) + +func NewEventWriter(target io.Writer, opts ...EventWriterOption) Writer { + writer := &eventWriter{ + target: target, + byteOrder: defaultOrder, + lengthPool: &sync.Pool{ + New: func() interface{} { + return make([]byte, lengthBufferSize) + }, + }, + } + + for _, opt := range opts { + opt(writer) + } + + return writer +} + +type eventWriter struct { + lengthPool *sync.Pool + target io.Writer + byteOrder binary.ByteOrder +} + +func (e eventWriter) Write(ev *Event) (err error) { + if ev == nil { + return ErrValueMostNotBeNil + } + var bytes []byte + + if bytes, err = proto.Marshal(ev.ProtoMessage()); err != nil { + return + } + buf := e.lengthPool.Get().([]byte) + e.byteOrder.PutUint32(buf, uint32(len(bytes))) + + if _, err = e.target.Write(buf); err != nil { + return + } + if _, err = e.target.Write(bytes); err != nil { + return + } + if syncerInst, ok := e.target.(syncer); ok { + err = syncerInst.Sync() + } + + return +} + +func (e eventWriter) Close() error { + if closer, isCloser := e.target.(io.Closer); isCloser { + return closer.Close() + } + return nil +} diff --git a/pkg/audit/writer_sink.go b/pkg/audit/writer_sink.go new file mode 100644 index 0000000..803d29a --- /dev/null +++ b/pkg/audit/writer_sink.go @@ -0,0 +1,47 @@ +package audit + +type WriterSinkOption func(sink *writerCloserSink) + +var ( + WithCloseOnExit WriterSinkOption = func(sink *writerCloserSink) { + sink.closeOnExit = true + } +) + +func NewWriterSink(name string, target Writer, opts ...WriterSinkOption) Sink { + sink := &writerCloserSink{ + name: name, + target: target, + } + + for _, opt := range opts { + opt(sink) + } + + return sink +} + +type writerCloserSink struct { + name string + target Writer + closeOnExit bool +} + +type syncer interface { + Sync() error +} + +func (f writerCloserSink) Name() string { + return f.name +} + +func (f writerCloserSink) OnSubscribe(evs <-chan Event) { + go func(target Writer, closeOnExit bool, evs <-chan Event) { + for ev := range evs { + _ = target.Write(&ev) + } + if closeOnExit { + _ = target.Close() + } + }(f.target, f.closeOnExit, evs) +} diff --git a/pkg/audit/writer_sink_test.go b/pkg/audit/writer_sink_test.go new file mode 100644 index 0000000..395cc5a --- /dev/null +++ b/pkg/audit/writer_sink_test.go @@ -0,0 +1,72 @@ +package audit_test + +import ( + "sync" + "testing" + "time" + + "github.com/golang/mock/gomock" + audit_mock "gitlab.com/inetmock/inetmock/internal/mock/audit" + "gitlab.com/inetmock/inetmock/pkg/audit" + "gitlab.com/inetmock/inetmock/pkg/logging" +) + +func Test_writerCloserSink_OnSubscribe(t *testing.T) { + type testCase struct { + name string + events []audit.Event + } + tests := []testCase{ + { + name: "Get a single event", + events: testEvents[:1], + }, + { + name: "Get multiple events", + events: testEvents, + }, + } + scenario := func(tt testCase) func(t *testing.T) { + return func(t *testing.T) { + wg := new(sync.WaitGroup) + wg.Add(len(tt.events)) + + ctrl := gomock.NewController(t) + t.Cleanup(ctrl.Finish) + writerMock := audit_mock.NewMockWriter(ctrl) + writerMock. + EXPECT(). + Write(gomock.Any()). + Do(func(_ *audit.Event) { + wg.Done() + }). + Times(len(tt.events)) + + writerCloserSink := audit.NewWriterSink("WriterMock", writerMock, audit.WithCloseOnExit) + var evs audit.EventStream + var err error + + if evs, err = audit.NewEventStream(logging.CreateTestLogger(t)); err != nil { + t.Errorf("NewEventStream() error = %v", err) + } + + if err = evs.RegisterSink(writerCloserSink); err != nil { + t.Errorf("RegisterSink() error = %v", err) + } + + for _, ev := range tt.events { + evs.Emit(ev) + } + + select { + case <-time.After(100 * time.Millisecond): + t.Errorf("not all events recorded in time") + case <-waitGroupDone(wg): + } + } + } + + for _, tt := range tests { + t.Run(tt.name, scenario(tt)) + } +} diff --git a/pkg/audit/writer_test.go b/pkg/audit/writer_test.go new file mode 100644 index 0000000..ce8264b --- /dev/null +++ b/pkg/audit/writer_test.go @@ -0,0 +1,117 @@ +//go:generate mockgen -source=$GOFILE -destination=./../../internal/mock/audit/writercloser.mock.go -package=audit_mock + +package audit_test + +import ( + "encoding/binary" + "encoding/hex" + "io" + "testing" + + "github.com/golang/mock/gomock" + audit_mock "gitlab.com/inetmock/inetmock/internal/mock/audit" + "gitlab.com/inetmock/inetmock/pkg/audit" +) + +type WriterCloserSyncer interface { + io.WriteCloser + Sync() error +} + +func Test_eventWriter_Write(t *testing.T) { + type fields struct { + order binary.ByteOrder + } + type args struct { + evs []audit.Event + } + type testCase struct { + name string + fields fields + args args + wantErr bool + } + tests := []testCase{ + { + name: "Write a single event - little endian", + fields: fields{ + order: binary.LittleEndian, + }, + args: args{ + evs: testEvents[:1], + }, + wantErr: false, + }, + { + name: "Write a single event - big endian", + fields: fields{ + order: binary.BigEndian, + }, + args: args{ + evs: testEvents[:1], + }, + wantErr: false, + }, + { + name: "Write multiple events - little endian", + fields: fields{ + order: binary.LittleEndian, + }, + args: args{ + evs: testEvents, + }, + wantErr: false, + }, + { + name: "Write multiple events - big endian", + fields: fields{ + order: binary.BigEndian, + }, + args: args{ + evs: testEvents, + }, + wantErr: false, + }, + } + scenario := func(tt testCase) func(t *testing.T) { + return func(t *testing.T) { + ctrl := gomock.NewController(t) + t.Cleanup(ctrl.Finish) + writerMock := audit_mock.NewMockWriterCloserSyncer(ctrl) + calls := make([]*gomock.Call, 0) + for i := 0; i < len(tt.args.evs); i++ { + calls = append(calls, + writerMock. + EXPECT(). + Write(gomock.Any()). + Do(func(data []byte) { + t.Logf("got payload = %s", hex.EncodeToString(data)) + t.Logf("got length %d", tt.fields.order.Uint32(data)) + }), + writerMock. + EXPECT(). + Write(gomock.Any()). + Do(func(data []byte) { + t.Logf("got payload = %s", hex.EncodeToString(data)) + }), + writerMock. + EXPECT(). + Sync(), + ) + } + gomock.InOrder(calls...) + + e := audit.NewEventWriter(writerMock, audit.WithWriterByteOrder(tt.fields.order)) + + for _, ev := range tt.args.evs { + if err := e.Write(&ev); (err != nil) != tt.wantErr { + t.Errorf("Write() error = %v, wantErr %v", err, tt.wantErr) + } + } + } + } + + for _, tt := range tests { + t.Run(tt.name, scenario(tt)) + } +} diff --git a/plugins/tls_interceptor/register.go b/plugins/tls_interceptor/register.go index d354da5..e8ec9cb 100644 --- a/plugins/tls_interceptor/register.go +++ b/plugins/tls_interceptor/register.go @@ -40,7 +40,7 @@ func AddTLSInterceptor(registry api.HandlerRegistry) (err error) { registry.RegisterHandler(name, func() api.ProtocolHandler { return &tlsInterceptor{ logger: logger, - currentConnectionsCount: &sync.WaitGroup{}, + currentConnectionsCount: new(sync.WaitGroup), currentConnections: make(map[uuid.UUID]*proxyConn), connectionsMutex: &sync.Mutex{}, } From 0468c9367174ce23abddc6eeb4f6c4de9c1734b4 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Mon, 4 Jan 2021 17:52:21 +0100 Subject: [PATCH 04/26] Integrate into handlers --- internal/app/app.go | 27 +++++++++- pkg/api/protocol_handler.go | 2 + pkg/audit/event_stream.go | 33 ++++++++---- pkg/audit/event_stream_test.go | 4 +- pkg/audit/log_sink.go | 1 - pkg/audit/options.go | 36 +++++++++---- plugins/http_mock/conn_context.go | 35 ++++++++++++ plugins/http_mock/handler.go | 7 ++- plugins/http_mock/handler_test.go | 8 +++ plugins/http_mock/regex_router.go | 89 ++++++++++++++++++++++--------- 10 files changed, 190 insertions(+), 52 deletions(-) create mode 100644 plugins/http_mock/conn_context.go diff --git a/internal/app/app.go b/internal/app/app.go index ce8d1ff..6c4adfe 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/cobra" "gitlab.com/inetmock/inetmock/internal/endpoints" "gitlab.com/inetmock/inetmock/pkg/api" + "gitlab.com/inetmock/inetmock/pkg/audit" "gitlab.com/inetmock/inetmock/pkg/cert" "gitlab.com/inetmock/inetmock/pkg/config" "gitlab.com/inetmock/inetmock/pkg/health" @@ -47,6 +48,7 @@ type app struct { registry api.HandlerRegistry ctx context.Context cancel context.CancelFunc + eventStream audit.EventStream } func (a *app) MustRun() { @@ -82,6 +84,10 @@ func (a app) EndpointManager() endpoints.EndpointManager { return a.endpointManager } +func (a app) Audit() audit.Emitter { + return a.eventStream +} + func (a app) HandlerRegistry() api.HandlerRegistry { return a.registry } @@ -137,15 +143,32 @@ func NewApp(registrations ...api.Registration) (inetmockApp App, err error) { return } + a.endpointManager = endpoints.NewEndpointManager( + a.registry, + a.Logger().Named("EndpointManager"), + a.checker, + a, + ) + a.cfg = config.CreateConfig(cmd.Flags()) if err = a.cfg.ReadConfig(configFilePath); err != nil { return } - a.certStore, err = cert.NewDefaultStore(a.cfg, a.rootLogger) - a.endpointManager = endpoints.NewEndpointManager(a.registry, a.Logger().Named("EndpointManager"), a.checker, a) + if a.certStore, err = cert.NewDefaultStore(a.cfg, a.rootLogger); err != nil { + return + } + a.eventStream, err = audit.NewEventStream( + a.Logger().Named("EventStream"), + audit.WithSinkBufferSize(10), + ) + if err != nil { + return + } + + err = a.eventStream.RegisterSink(audit.NewLogSink(a.Logger().Named("LogSink"))) return } diff --git a/pkg/api/protocol_handler.go b/pkg/api/protocol_handler.go index 540ae77..f927570 100644 --- a/pkg/api/protocol_handler.go +++ b/pkg/api/protocol_handler.go @@ -4,6 +4,7 @@ package api import ( "context" + "gitlab.com/inetmock/inetmock/pkg/audit" "gitlab.com/inetmock/inetmock/pkg/cert" "gitlab.com/inetmock/inetmock/pkg/config" "gitlab.com/inetmock/inetmock/pkg/logging" @@ -12,6 +13,7 @@ import ( type PluginContext interface { Logger() logging.Logger CertStore() cert.Store + Audit() audit.Emitter } type ProtocolHandler interface { diff --git a/pkg/audit/event_stream.go b/pkg/audit/event_stream.go index bc22331..51f6c7c 100644 --- a/pkg/audit/event_stream.go +++ b/pkg/audit/event_stream.go @@ -16,13 +16,18 @@ func init() { type eventStream struct { logger logging.Logger buffer chan *Event - sinks map[string]chan Event + sinks map[string]*registeredSink lock sync.Locker idGenerator *snowflake.Node sinkBufferSize int sinkConsumptionTimeout time.Duration } +type registeredSink struct { + downstream chan Event + lock sync.Locker +} + func NewEventStream(logger logging.Logger, options ...EventStreamOption) (es EventStream, err error) { cfg := newEventStreamCfg() @@ -39,7 +44,7 @@ func NewEventStream(logger logging.Logger, options ...EventStreamOption) (es Eve generatorIdx++ underlying := &eventStream{ logger: logger, - sinks: make(map[string]chan Event), + sinks: make(map[string]*registeredSink), buffer: make(chan *Event, cfg.bufferSize), sinkBufferSize: cfg.sinkBuffersize, sinkConsumptionTimeout: cfg.sinkConsumptionTimeout, @@ -47,7 +52,10 @@ func NewEventStream(logger logging.Logger, options ...EventStreamOption) (es Eve lock: &sync.Mutex{}, } - go underlying.distribute() + // start distribute workers + for i := 0; i < cfg.distributeParallelization; i++ { + go underlying.distribute() + } es = underlying @@ -71,9 +79,12 @@ func (e *eventStream) RegisterSink(s Sink) error { return ErrSinkAlreadyRegistered } - downstream := make(chan Event, e.sinkBufferSize) - s.OnSubscribe(downstream) - e.sinks[name] = downstream + rs := ®isteredSink{ + downstream: make(chan Event, e.sinkBufferSize), + lock: new(sync.Mutex), + } + s.OnSubscribe(rs.downstream) + e.sinks[name] = rs return nil } @@ -86,22 +97,24 @@ func (e eventStream) Sinks() (sinks []string) { func (e *eventStream) Close() error { close(e.buffer) - for _, downstream := range e.sinks { - close(downstream) + for _, rs := range e.sinks { + close(rs.downstream) } return nil } func (e *eventStream) distribute() { for ev := range e.buffer { - for name, s := range e.sinks { + for name, rs := range e.sinks { + rs.lock.Lock() e.logger.Debug("notify sink", zap.String("sink", name)) select { - case s <- *ev: + case rs.downstream <- *ev: e.logger.Debug("pushed event to sink channel") case <-time.After(e.sinkConsumptionTimeout): e.logger.Warn("sink consummation timed out") } + rs.lock.Unlock() } } } diff --git a/pkg/audit/event_stream_test.go b/pkg/audit/event_stream_test.go index 17732e3..4e0cd70 100644 --- a/pkg/audit/event_stream_test.go +++ b/pkg/audit/event_stream_test.go @@ -169,7 +169,7 @@ func Test_eventStream_Emit(t *testing.T) { name: "Expect to get a single event", subscribe: true, args: args{ - opts: []audit.EventStreamOption{audit.WithBufferSize(10)}, + opts: []audit.EventStreamOption{}, evs: testEvents[:1], }, }, @@ -177,7 +177,7 @@ func Test_eventStream_Emit(t *testing.T) { name: "Expect to get multiple events", subscribe: true, args: args{ - opts: []audit.EventStreamOption{audit.WithBufferSize(10)}, + opts: []audit.EventStreamOption{}, evs: testEvents, }, }, diff --git a/pkg/audit/log_sink.go b/pkg/audit/log_sink.go index a6e07f1..c6ce90d 100644 --- a/pkg/audit/log_sink.go +++ b/pkg/audit/log_sink.go @@ -46,7 +46,6 @@ func (l logSink) OnSubscribe(evs <-chan Event) { zap.Uint16("source_port", ev.SourcePort), zap.String("destination_ip", ev.DestinationIP.String()), zap.Uint16("destination_port", ev.DestinationPort), - zap.Any("details", ev.ProtocolDetails), ) } }(l.logger, evs) diff --git a/pkg/audit/options.go b/pkg/audit/options.go index 24ee746..76b737b 100644 --- a/pkg/audit/options.go +++ b/pkg/audit/options.go @@ -1,6 +1,9 @@ package audit -import "time" +import ( + "runtime" + "time" +) const ( defaultEventStreamBufferSize = 100 @@ -9,8 +12,9 @@ const ( ) var ( - generatorIdx int64 = 1 - WithBufferSize = func(bufferSize int) EventStreamOption { + generatorIdx int64 = 1 + defaultDistributeParallelization = runtime.NumCPU() / 2 + WithBufferSize = func(bufferSize int) EventStreamOption { return func(cfg *eventStreamCfg) { cfg.bufferSize = bufferSize } @@ -30,23 +34,33 @@ var ( cfg.sinkConsumptionTimeout = timeout } } + WithDistributeParallelization = func(parallelization int) EventStreamOption { + return func(cfg *eventStreamCfg) { + if parallelization <= 0 || parallelization > runtime.NumCPU() { + return + } + cfg.distributeParallelization = parallelization + } + } ) type EventStreamOption func(cfg *eventStreamCfg) type eventStreamCfg struct { - bufferSize int - sinkBuffersize int - generatorIndex int64 - sinkConsumptionTimeout time.Duration + bufferSize int + sinkBuffersize int + generatorIndex int64 + distributeParallelization int + sinkConsumptionTimeout time.Duration } func newEventStreamCfg() eventStreamCfg { cfg := eventStreamCfg{ - generatorIndex: generatorIdx, - sinkBuffersize: defaultSinkBufferSize, - bufferSize: defaultEventStreamBufferSize, - sinkConsumptionTimeout: defaultSinkConsumptionTimeout, + generatorIndex: generatorIdx, + sinkBuffersize: defaultSinkBufferSize, + bufferSize: defaultEventStreamBufferSize, + sinkConsumptionTimeout: defaultSinkConsumptionTimeout, + distributeParallelization: defaultDistributeParallelization, } generatorIdx++ diff --git a/plugins/http_mock/conn_context.go b/plugins/http_mock/conn_context.go new file mode 100644 index 0000000..b22af86 --- /dev/null +++ b/plugins/http_mock/conn_context.go @@ -0,0 +1,35 @@ +package http_mock + +import ( + "context" + "net" +) + +type httpContextKey string + +const ( + remoteAddrKey httpContextKey = "RemoteAddr" + localAddrKey httpContextKey = "LocalAddr" +) + +func StoreConnPropertiesInContext(ctx context.Context, c net.Conn) context.Context { + ctx = context.WithValue(ctx, remoteAddrKey, c.RemoteAddr()) + ctx = context.WithValue(ctx, localAddrKey, c.LocalAddr()) + return ctx +} + +func LocalAddr(ctx context.Context) net.Addr { + val := ctx.Value(localAddrKey) + if val == nil { + return nil + } + return val.(net.Addr) +} + +func RemoteAddr(ctx context.Context) net.Addr { + val := ctx.Value(remoteAddrKey) + if val == nil { + return nil + } + return val.(net.Addr) +} diff --git a/plugins/http_mock/handler.go b/plugins/http_mock/handler.go index c87bd2a..f1acb23 100644 --- a/plugins/http_mock/handler.go +++ b/plugins/http_mock/handler.go @@ -40,9 +40,14 @@ func (p *httpHandler) Start(ctx api.PluginContext, config config.HandlerConfig) router := &RegexpHandler{ logger: p.logger, + emitter: ctx.Audit(), handlerName: config.HandlerName, } - p.server = &http.Server{Addr: config.ListenAddr(), Handler: router} + p.server = &http.Server{ + Addr: config.ListenAddr(), + Handler: router, + ConnContext: StoreConnPropertiesInContext, + } for _, rule := range options.Rules { router.setupRoute(rule) diff --git a/plugins/http_mock/handler_test.go b/plugins/http_mock/handler_test.go index f60d031..804943a 100644 --- a/plugins/http_mock/handler_test.go +++ b/plugins/http_mock/handler_test.go @@ -14,6 +14,7 @@ import ( "github.com/golang/mock/gomock" "github.com/spf13/viper" api_mock "gitlab.com/inetmock/inetmock/internal/mock/api" + audit_mock "gitlab.com/inetmock/inetmock/internal/mock/audit" "gitlab.com/inetmock/inetmock/pkg/api" "gitlab.com/inetmock/inetmock/pkg/config" "gitlab.com/inetmock/inetmock/pkg/logging" @@ -72,11 +73,18 @@ func setupHandler(b *testing.B, ctrl *gomock.Controller, listenPort uint16) (api b.Error("handler not registered") } + emitter := audit_mock.NewMockEmitter(ctrl) + emitter.EXPECT().Emit(gomock.Any()).AnyTimes() + mockApp := api_mock.NewMockPluginContext(ctrl) mockApp.EXPECT(). Logger(). Return(testLogger) + mockApp.EXPECT(). + Audit(). + Return(emitter) + v := viper.New() v.Set("rules", []map[string]string{ { diff --git a/plugins/http_mock/regex_router.go b/plugins/http_mock/regex_router.go index 3de7ea4..9b643d1 100644 --- a/plugins/http_mock/regex_router.go +++ b/plugins/http_mock/regex_router.go @@ -1,13 +1,16 @@ package http_mock import ( - "bytes" + "net" "net/http" "strconv" + "strings" "github.com/prometheus/client_golang/prometheus" + "gitlab.com/inetmock/inetmock/pkg/audit" "gitlab.com/inetmock/inetmock/pkg/logging" "go.uber.org/zap" + "google.golang.org/protobuf/types/known/anypb" ) type route struct { @@ -19,16 +22,13 @@ type RegexpHandler struct { handlerName string logger logging.Logger routes []*route + emitter audit.Emitter } func (h *RegexpHandler) Handler(rule targetRule, handler http.Handler) { h.routes = append(h.routes, &route{rule, handler}) } -func (h *RegexpHandler) HandleFunc(rule targetRule, handler func(http.ResponseWriter, *http.Request)) { - h.routes = append(h.routes, &route{rule, http.HandlerFunc(handler)}) -} - func (h *RegexpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { timer := prometheus.NewTimer(requestDurationHistogram.WithLabelValues(h.handlerName)) defer timer.ObserveDuration() @@ -53,25 +53,64 @@ func (h *RegexpHandler) setupRoute(rule targetRule) { zap.String("response", rule.Response()), ) - h.Handler(rule, createHandlerForTarget(h.logger, rule.response)) -} - -func createHandlerForTarget(logger logging.Logger, targetPath string) http.Handler { - return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - headerWriter := &bytes.Buffer{} - request.Header.Write(headerWriter) - - logger.Info( - "Handling request", - zap.String("source", request.RemoteAddr), - zap.String("host", request.Host), - zap.String("method", request.Method), - zap.String("protocol", request.Proto), - zap.String("path", request.RequestURI), - zap.String("response", targetPath), - zap.Reflect("headers", request.Header), - ) - - http.ServeFile(writer, request, targetPath) + h.Handler(rule, emittingFileHandler{ + emitter: h.emitter, + targetPath: rule.response, }) } + +type emittingFileHandler struct { + emitter audit.Emitter + targetPath string +} + +func (f emittingFileHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + f.emitter.Emit(eventFromRequest(request)) + http.ServeFile(writer, request, f.targetPath) +} + +func eventFromRequest(request *http.Request) audit.Event { + details := audit.HTTPDetails{ + Method: request.Method, + Host: request.Host, + URI: request.RequestURI, + Proto: request.Proto, + Headers: request.Header, + } + var wire *anypb.Any + if w, err := details.MarshalToWireFormat(); err == nil { + wire = w + } + + localIP, localPort := ipPortFromAddr(LocalAddr(request.Context())) + remoteIP, remotePort := ipPortFromAddr(RemoteAddr(request.Context())) + + return audit.Event{ + Transport: audit.TransportProtocol_TCP, + Application: audit.AppProtocol_HTTP, + SourceIP: remoteIP, + DestinationIP: localIP, + SourcePort: remotePort, + DestinationPort: localPort, + ProtocolDetails: wire, + } +} + +func ipPortFromAddr(addr net.Addr) (ip net.IP, port uint16) { + if tcpAddr, isTCPAddr := addr.(*net.TCPAddr); isTCPAddr { + return tcpAddr.IP, uint16(tcpAddr.Port) + } + + ipPortSplit := strings.Split(addr.String(), ":") + if len(ipPortSplit) != 2 { + return + } + + ip = net.ParseIP(ipPortSplit[0]) + + if p, err := strconv.Atoi(ipPortSplit[1]); err == nil { + port = uint16(p) + } + + return +} From 6c448fd318485db4c405cfaf68a97830951c3040 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Thu, 7 Jan 2021 22:00:12 +0100 Subject: [PATCH 05/26] Add DNS details --- api/proto/pkg/audit/details/dns_details.proto | 42 ++++++++ .../audit/{ => details}/http_details.proto | 4 +- pkg/audit/details/dns_details.go | 47 ++++++++ pkg/audit/{ => details}/http_details.go | 23 +++- pkg/audit/event.go | 102 ++++++++++++------ pkg/audit/event_stream_test.go | 30 ++---- pkg/audit/event_test.go | 61 +++++++++++ pkg/audit/ip_conversion.go | 33 ++++++ pkg/audit/log_sink.go | 1 + pkg/audit/log_sink_test.go | 4 +- pkg/audit/reader_test.go | 6 +- pkg/audit/writer_sink_test.go | 4 +- pkg/audit/writer_test.go | 4 +- plugins/dns_mock/handler.go | 11 +- plugins/dns_mock/regex_handler.go | 100 +++++++++++------ plugins/dns_mock/register.go | 14 +-- plugins/http_mock/protocol_options.go | 2 +- plugins/http_mock/regex_router.go | 45 ++------ 18 files changed, 383 insertions(+), 150 deletions(-) create mode 100644 api/proto/pkg/audit/details/dns_details.proto rename api/proto/pkg/audit/{ => details}/http_details.proto (79%) create mode 100644 pkg/audit/details/dns_details.go rename pkg/audit/{ => details}/http_details.go (57%) create mode 100644 pkg/audit/event_test.go create mode 100644 pkg/audit/ip_conversion.go diff --git a/api/proto/pkg/audit/details/dns_details.proto b/api/proto/pkg/audit/details/dns_details.proto new file mode 100644 index 0000000..1575e32 --- /dev/null +++ b/api/proto/pkg/audit/details/dns_details.proto @@ -0,0 +1,42 @@ +syntax = "proto3"; + +option go_package = "gitlab.com/inetmock/inetmock/pkg/audit/details"; +option java_multiple_files = true; +option java_package = "com.github.baez90.inetmock.audit.details"; +option java_outer_classname = "HandlerEventProto"; + +package inetmock.audit; + +enum DNSOpCode { + Query = 0; + Status = 2; + Notify = 4; + Update = 5; +} + +enum ResourceRecordType { + UnknownRR = 0; + A = 1; + NS = 2; + CNAME = 5; + SOA = 6; + PTR = 12; + HINFO = 13; + MINFO = 14; + MX = 15; + TXT = 16; + RP = 17; + AAAA = 28; + SRV = 33; + NAPTR = 35; +} + +message DNSQuestionEntity { + ResourceRecordType type = 1; + string name = 2; +} + +message DNSDetailsEntity { + DNSOpCode opcode = 1; + repeated DNSQuestionEntity questions = 2; +} \ No newline at end of file diff --git a/api/proto/pkg/audit/http_details.proto b/api/proto/pkg/audit/details/http_details.proto similarity index 79% rename from api/proto/pkg/audit/http_details.proto rename to api/proto/pkg/audit/details/http_details.proto index a771024..ffd5ac0 100644 --- a/api/proto/pkg/audit/http_details.proto +++ b/api/proto/pkg/audit/details/http_details.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -option go_package = "gitlab.com/inetmock/inetmock/pkg/audit"; +option go_package = "gitlab.com/inetmock/inetmock/pkg/audit/details"; option java_multiple_files = true; -option java_package = "com.github.baez90.inetmock.audit"; +option java_package = "com.github.baez90.inetmock.audit.details"; option java_outer_classname = "HandlerEventProto"; package inetmock.audit; diff --git a/pkg/audit/details/dns_details.go b/pkg/audit/details/dns_details.go new file mode 100644 index 0000000..91397dd --- /dev/null +++ b/pkg/audit/details/dns_details.go @@ -0,0 +1,47 @@ +package details + +import ( + "google.golang.org/protobuf/types/known/anypb" +) + +func NewDNSFromWireFormat(entity *DNSDetailsEntity) DNS { + d := DNS{ + OPCode: entity.Opcode, + } + + for _, q := range entity.Questions { + d.Questions = append(d.Questions, DNSQuestion{ + RRType: q.Type, + Name: q.Name, + }) + } + + return d +} + +type DNSQuestion struct { + RRType ResourceRecordType + Name string +} + +type DNS struct { + OPCode DNSOpCode + Questions []DNSQuestion +} + +func (d DNS) MarshalToWireFormat() (any *anypb.Any, err error) { + detailsEntity := &DNSDetailsEntity{ + Opcode: d.OPCode, + } + + for _, q := range d.Questions { + detailsEntity.Questions = append(detailsEntity.Questions, &DNSQuestionEntity{ + Type: q.RRType, + Name: q.Name, + }) + } + + any, err = anypb.New(detailsEntity) + + return +} diff --git a/pkg/audit/http_details.go b/pkg/audit/details/http_details.go similarity index 57% rename from pkg/audit/http_details.go rename to pkg/audit/details/http_details.go index b755a45..bee3928 100644 --- a/pkg/audit/http_details.go +++ b/pkg/audit/details/http_details.go @@ -1,4 +1,4 @@ -package audit +package details import ( "net/http" @@ -7,7 +7,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" ) -type HTTPDetails struct { +type HTTP struct { Method string Host string URI string @@ -15,7 +15,24 @@ type HTTPDetails struct { Headers http.Header } -func (d HTTPDetails) MarshalToWireFormat() (any *anypb.Any, err error) { +func NewHTTPFromWireFormat(entity *HTTPDetailsEntity) HTTP { + headers := http.Header{} + for name, values := range entity.Headers { + for idx := range values.Values { + headers.Add(name, values.Values[idx]) + } + } + + return HTTP{ + Method: entity.Method.String(), + Host: entity.Host, + URI: entity.Uri, + Proto: entity.Proto, + Headers: headers, + } +} + +func (d HTTP) MarshalToWireFormat() (any *anypb.Any, err error) { var method = HTTPMethod_GET if methodValue, known := HTTPMethod_value[strings.ToUpper(d.Method)]; known { method = HTTPMethod(methodValue) diff --git a/pkg/audit/event.go b/pkg/audit/event.go index 84fb7d1..0ec4aa5 100644 --- a/pkg/audit/event.go +++ b/pkg/audit/event.go @@ -1,11 +1,12 @@ package audit import ( - "encoding/binary" - "math/big" "net" + "strconv" + "strings" "time" + "gitlab.com/inetmock/inetmock/pkg/audit/details" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/timestamppb" @@ -24,35 +25,23 @@ type Event struct { DestinationIP net.IP SourcePort uint16 DestinationPort uint16 - ProtocolDetails *anypb.Any + ProtocolDetails Details TLS *TLSDetails } func (e *Event) ProtoMessage() proto.Message { var sourceIP isEventEntity_SourceIP if ipv4 := e.SourceIP.To4(); ipv4 != nil { - if len(ipv4) == 16 { - sourceIP = &EventEntity_SourceIPv4{SourceIPv4: binary.BigEndian.Uint32(ipv4[12:16])} - } else { - sourceIP = &EventEntity_SourceIPv4{SourceIPv4: binary.BigEndian.Uint32(ipv4)} - } + sourceIP = &EventEntity_SourceIPv4{SourceIPv4: ipv4ToUint32(ipv4)} } else { - ipv6 := big.NewInt(0) - ipv6.SetBytes(e.SourceIP) - sourceIP = &EventEntity_SourceIPv6{SourceIPv6: ipv6.Uint64()} + sourceIP = &EventEntity_SourceIPv6{SourceIPv6: ipv6ToBytes(e.SourceIP)} } var destinationIP isEventEntity_DestinationIP if ipv4 := e.DestinationIP.To4(); ipv4 != nil { - if len(ipv4) == 16 { - destinationIP = &EventEntity_DestinationIPv4{DestinationIPv4: binary.BigEndian.Uint32(ipv4[12:16])} - } else { - destinationIP = &EventEntity_DestinationIPv4{DestinationIPv4: binary.BigEndian.Uint32(ipv4)} - } + destinationIP = &EventEntity_DestinationIPv4{DestinationIPv4: ipv4ToUint32(ipv4)} } else { - ipv6 := big.NewInt(0) - ipv6.SetBytes(e.SourceIP) - destinationIP = &EventEntity_DestinationIPv6{DestinationIPv6: ipv6.Uint64()} + destinationIP = &EventEntity_DestinationIPv6{DestinationIPv6: ipv6ToBytes(e.DestinationIP)} } var tlsDetails *TLSDetailsEntity = nil @@ -60,6 +49,13 @@ func (e *Event) ProtoMessage() proto.Message { tlsDetails = e.TLS.ProtoMessage() } + var detailsEntity *anypb.Any = nil + if e.ProtocolDetails != nil { + if any, err := e.ProtocolDetails.MarshalToWireFormat(); err == nil { + detailsEntity = any + } + } + return &EventEntity{ Id: e.ID, Timestamp: timestamppb.New(e.Timestamp), @@ -70,7 +66,7 @@ func (e *Event) ProtoMessage() proto.Message { SourcePort: uint32(e.SourcePort), DestinationPort: uint32(e.DestinationPort), Tls: tlsDetails, - ProtocolDetails: e.ProtocolDetails, + ProtocolDetails: detailsEntity, } } @@ -82,27 +78,33 @@ func (e *Event) ApplyDefaults(id int64) { } } +func (e *Event) SetSourceIPFromAddr(remoteAddr net.Addr) { + ip, port := parseIPPortFromAddr(remoteAddr) + e.SourceIP = ip + e.SourcePort = port +} + +func (e *Event) SetDestinationIPFromAddr(localAddr net.Addr) { + ip, port := parseIPPortFromAddr(localAddr) + e.DestinationIP = ip + e.DestinationPort = port +} + func NewEventFromProto(msg *EventEntity) (ev Event) { var sourceIP net.IP switch ip := msg.GetSourceIP().(type) { case *EventEntity_SourceIPv4: - buf := make([]byte, 4) - binary.BigEndian.PutUint32(buf, ip.SourceIPv4) - sourceIP = buf - sourceIP = sourceIP.To4() + sourceIP = uint32ToIP(ip.SourceIPv4) case *EventEntity_SourceIPv6: - sourceIP = big.NewInt(int64(ip.SourceIPv6)).Bytes() + sourceIP = uint64ToIP(ip.SourceIPv6) } var destinationIP net.IP switch ip := msg.GetDestinationIP().(type) { case *EventEntity_DestinationIPv4: - buf := make([]byte, 4) - binary.BigEndian.PutUint32(buf, ip.DestinationIPv4) - destinationIP = buf - destinationIP = destinationIP.To4() + destinationIP = uint32ToIP(ip.DestinationIPv4) case *EventEntity_DestinationIPv6: - destinationIP = big.NewInt(int64(ip.DestinationIPv6)).Bytes() + destinationIP = uint64ToIP(ip.DestinationIPv6) } ev = Event{ @@ -114,8 +116,46 @@ func NewEventFromProto(msg *EventEntity) (ev Event) { DestinationIP: destinationIP, SourcePort: uint16(msg.GetSourcePort()), DestinationPort: uint16(msg.GetDestinationPort()), - ProtocolDetails: msg.GetProtocolDetails(), + ProtocolDetails: guessDetailsFromApp(msg.GetProtocolDetails()), TLS: NewTLSDetailsFromProto(msg.GetTls()), } return } + +func parseIPPortFromAddr(addr net.Addr) (ip net.IP, port uint16) { + switch a := addr.(type) { + case *net.TCPAddr: + return a.IP, uint16(a.Port) + case *net.UDPAddr: + return a.IP, uint16(a.Port) + case *net.UnixAddr: + return + default: + ipPortSplit := strings.Split(addr.String(), ":") + if len(ipPortSplit) != 2 { + return + } + + ip = net.ParseIP(ipPortSplit[0]) + if p, err := strconv.Atoi(ipPortSplit[1]); err == nil { + port = uint16(p) + } + return + } +} + +func guessDetailsFromApp(any *anypb.Any) Details { + var detailsProto proto.Message + var err error + if detailsProto, err = any.UnmarshalNew(); err != nil { + return nil + } + switch any.TypeUrl { + case "type.googleapis.com/inetmock.audit.HTTPDetailsEntity": + return details.NewHTTPFromWireFormat(detailsProto.(*details.HTTPDetailsEntity)) + case "type.googleapis.com/inetmock.audit.DNSDetailsEntity": + return details.NewDNSFromWireFormat(detailsProto.(*details.DNSDetailsEntity)) + default: + return nil + } +} diff --git a/pkg/audit/event_stream_test.go b/pkg/audit/event_stream_test.go index 4e0cd70..8e25eca 100644 --- a/pkg/audit/event_stream_test.go +++ b/pkg/audit/event_stream_test.go @@ -9,20 +9,20 @@ import ( "time" "gitlab.com/inetmock/inetmock/pkg/audit" + "gitlab.com/inetmock/inetmock/pkg/audit/details" "gitlab.com/inetmock/inetmock/pkg/logging" - "google.golang.org/protobuf/types/known/anypb" ) var ( defaultSink = &testSink{ name: "test defaultSink", } - testEvents = []audit.Event{ + testEvents = []*audit.Event{ { Transport: audit.TransportProtocol_TCP, Application: audit.AppProtocol_HTTP, - SourceIP: net.ParseIP("127.0.0.1"), - DestinationIP: net.ParseIP("127.0.0.1"), + SourceIP: net.ParseIP("127.0.0.1").To4(), + DestinationIP: net.ParseIP("127.0.0.1").To4(), SourcePort: 32344, DestinationPort: 80, TLS: &audit.TLSDetails{ @@ -30,7 +30,7 @@ var ( CipherSuite: tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, ServerName: "localhost", }, - ProtocolDetails: mustMarshalToWireformat(audit.HTTPDetails{ + ProtocolDetails: details.HTTP{ Method: "GET", Host: "localhost", URI: "http://localhost/asdf", @@ -38,13 +38,13 @@ var ( Headers: http.Header{ "Accept": []string{"application/json"}, }, - }), + }, }, { Transport: audit.TransportProtocol_TCP, Application: audit.AppProtocol_DNS, - SourceIP: net.ParseIP("::1"), - DestinationIP: net.ParseIP("::1"), + SourceIP: net.ParseIP("::1").To16(), + DestinationIP: net.ParseIP("::1").To16(), SourcePort: 32344, DestinationPort: 80, }, @@ -80,14 +80,6 @@ func wgMockSink(t testing.TB, wg *sync.WaitGroup) audit.Sink { } } -func mustMarshalToWireformat(d audit.Details) *anypb.Any { - any, err := d.MarshalToWireFormat() - if err != nil { - panic(err) - } - return any -} - func Test_eventStream_RegisterSink(t *testing.T) { type args struct { s audit.Sink @@ -156,7 +148,7 @@ func Test_eventStream_RegisterSink(t *testing.T) { func Test_eventStream_Emit(t *testing.T) { type args struct { - evs []audit.Event + evs []*audit.Event opts []audit.EventStreamOption } type testCase struct { @@ -215,9 +207,9 @@ func Test_eventStream_Emit(t *testing.T) { } } - go func(evs []audit.Event, wg *sync.WaitGroup) { + go func(evs []*audit.Event, wg *sync.WaitGroup) { for _, ev := range evs { - e.Emit(ev) + e.Emit(*ev) wg.Done() } }(tt.args.evs, emittedWaitGroup) diff --git a/pkg/audit/event_test.go b/pkg/audit/event_test.go new file mode 100644 index 0000000..b1aaebc --- /dev/null +++ b/pkg/audit/event_test.go @@ -0,0 +1,61 @@ +package audit + +import ( + "net/http" + "reflect" + "testing" + + "gitlab.com/inetmock/inetmock/pkg/audit/details" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +func Test_guessDetailsFromApp(t *testing.T) { + + mustAny := func(msg proto.Message) *anypb.Any { + a, err := anypb.New(msg) + if err != nil { + panic(a) + } + return a + } + + type args struct { + any *anypb.Any + } + type testCase struct { + name string + args args + want Details + } + tests := []testCase{ + { + name: "HTTP etails", + args: args{ + any: mustAny(&details.HTTPDetailsEntity{ + Method: details.HTTPMethod_GET, + Host: "localhost", + Uri: "http://localhost/asdf", + Proto: "HTTP", + }), + }, + want: details.HTTP{ + Method: "GET", + Host: "localhost", + URI: "http://localhost/asdf", + Proto: "HTTP", + Headers: http.Header{}, + }, + }, + } + scenario := func(tt testCase) func(t *testing.T) { + return func(t *testing.T) { + if got := guessDetailsFromApp(tt.args.any); !reflect.DeepEqual(got, tt.want) { + t.Errorf("guessDetailsFromApp() = %v, want %v", got, tt.want) + } + } + } + for _, tt := range tests { + t.Run(tt.name, scenario(tt)) + } +} diff --git a/pkg/audit/ip_conversion.go b/pkg/audit/ip_conversion.go new file mode 100644 index 0000000..7385eb3 --- /dev/null +++ b/pkg/audit/ip_conversion.go @@ -0,0 +1,33 @@ +package audit + +import ( + "encoding/binary" + "math/big" + "net" +) + +func ipv4ToUint32(ip net.IP) uint32 { + if len(ip) == 16 { + return binary.BigEndian.Uint32(ip[12:16]) + } + return binary.BigEndian.Uint32(ip) +} + +func ipv6ToBytes(ip net.IP) uint64 { + ipv6 := big.NewInt(0) + ipv6.SetBytes(ip) + return ipv6.Uint64() +} + +func uint32ToIP(i uint32) (ip net.IP) { + buf := make([]byte, 4) + binary.BigEndian.PutUint32(buf, i) + ip = buf + ip = ip.To4() + return +} + +func uint64ToIP(i uint64) (ip net.IP) { + ip = big.NewInt(int64(i)).FillBytes(make([]byte, 16)) + return +} diff --git a/pkg/audit/log_sink.go b/pkg/audit/log_sink.go index c6ce90d..a6e07f1 100644 --- a/pkg/audit/log_sink.go +++ b/pkg/audit/log_sink.go @@ -46,6 +46,7 @@ func (l logSink) OnSubscribe(evs <-chan Event) { zap.Uint16("source_port", ev.SourcePort), zap.String("destination_ip", ev.DestinationIP.String()), zap.Uint16("destination_port", ev.DestinationPort), + zap.Any("details", ev.ProtocolDetails), ) } }(l.logger, evs) diff --git a/pkg/audit/log_sink_test.go b/pkg/audit/log_sink_test.go index 085dc25..f201aa5 100644 --- a/pkg/audit/log_sink_test.go +++ b/pkg/audit/log_sink_test.go @@ -20,7 +20,7 @@ func Test_logSink_OnSubscribe(t *testing.T) { type testCase struct { name string fields fields - events []audit.Event + events []*audit.Event } tests := []testCase{ { @@ -93,7 +93,7 @@ func Test_logSink_OnSubscribe(t *testing.T) { } for _, ev := range tt.events { - evs.Emit(ev) + evs.Emit(*ev) } select { diff --git a/pkg/audit/reader_test.go b/pkg/audit/reader_test.go index 3335e79..e65ec7f 100644 --- a/pkg/audit/reader_test.go +++ b/pkg/audit/reader_test.go @@ -5,10 +5,10 @@ import ( "encoding/binary" "encoding/hex" "io" + "reflect" "testing" "gitlab.com/inetmock/inetmock/pkg/audit" - "google.golang.org/protobuf/proto" ) var ( @@ -36,7 +36,7 @@ func Test_eventReader_Read(t *testing.T) { type testCase struct { name string fields fields - wantEv audit.Event + wantEv *audit.Event wantErr bool } tests := []testCase{ @@ -86,7 +86,7 @@ func Test_eventReader_Read(t *testing.T) { return } - if err == nil && !proto.Equal(gotEv.ProtoMessage(), tt.wantEv.ProtoMessage()) { + if err == nil && !reflect.DeepEqual(gotEv, *tt.wantEv) { t.Errorf("Read() gotEv = %v, want %v", gotEv, tt.wantEv) } } diff --git a/pkg/audit/writer_sink_test.go b/pkg/audit/writer_sink_test.go index 395cc5a..483a5ac 100644 --- a/pkg/audit/writer_sink_test.go +++ b/pkg/audit/writer_sink_test.go @@ -14,7 +14,7 @@ import ( func Test_writerCloserSink_OnSubscribe(t *testing.T) { type testCase struct { name string - events []audit.Event + events []*audit.Event } tests := []testCase{ { @@ -55,7 +55,7 @@ func Test_writerCloserSink_OnSubscribe(t *testing.T) { } for _, ev := range tt.events { - evs.Emit(ev) + evs.Emit(*ev) } select { diff --git a/pkg/audit/writer_test.go b/pkg/audit/writer_test.go index ce8264b..17c2e88 100644 --- a/pkg/audit/writer_test.go +++ b/pkg/audit/writer_test.go @@ -23,7 +23,7 @@ func Test_eventWriter_Write(t *testing.T) { order binary.ByteOrder } type args struct { - evs []audit.Event + evs []*audit.Event } type testCase struct { name string @@ -104,7 +104,7 @@ func Test_eventWriter_Write(t *testing.T) { e := audit.NewEventWriter(writerMock, audit.WithWriterByteOrder(tt.fields.order)) for _, ev := range tt.args.evs { - if err := e.Write(&ev); (err != nil) != tt.wantErr { + if err := e.Write(ev); (err != nil) != tt.wantErr { t.Errorf("Write() error = %v, wantErr %v", err, tt.wantErr) } } diff --git a/plugins/dns_mock/handler.go b/plugins/dns_mock/handler.go index d07f7e0..0f9d372 100644 --- a/plugins/dns_mock/handler.go +++ b/plugins/dns_mock/handler.go @@ -15,22 +15,23 @@ type dnsHandler struct { dnsServer []*dns.Server } -func (d *dnsHandler) Start(_ api.PluginContext, config config.HandlerConfig) (err error) { +func (d *dnsHandler) Start(pluginCtx api.PluginContext, config config.HandlerConfig) (err error) { var options dnsOptions if options, err = loadFromConfig(config.Options); err != nil { return } listenAddr := config.ListenAddr() - d.logger = d.logger.With( + d.logger = pluginCtx.Logger().With( zap.String("handler_name", config.HandlerName), zap.String("address", listenAddr), ) handler := ®exHandler{ - handlerName: config.HandlerName, - fallback: options.Fallback, - logger: d.logger, + handlerName: config.HandlerName, + fallback: options.Fallback, + logger: pluginCtx.Logger(), + auditEmitter: pluginCtx.Audit(), } for _, rule := range options.Rules { diff --git a/plugins/dns_mock/regex_handler.go b/plugins/dns_mock/regex_handler.go index b6bc4c0..80dcb6a 100644 --- a/plugins/dns_mock/regex_handler.go +++ b/plugins/dns_mock/regex_handler.go @@ -1,35 +1,41 @@ package dns_mock import ( + "net" + "github.com/miekg/dns" "github.com/prometheus/client_golang/prometheus" + "gitlab.com/inetmock/inetmock/pkg/audit" + "gitlab.com/inetmock/inetmock/pkg/audit/details" "gitlab.com/inetmock/inetmock/pkg/logging" "go.uber.org/zap" ) type regexHandler struct { - handlerName string - routes []resolverRule - fallback ResolverFallback - logger logging.Logger + handlerName string + routes []resolverRule + fallback ResolverFallback + auditEmitter audit.Emitter + logger logging.Logger } func (rh *regexHandler) AddRule(rule resolverRule) { rh.routes = append(rh.routes, rule) } -func (rh regexHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { +func (rh *regexHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { timer := prometheus.NewTimer(requestDurationHistogram.WithLabelValues(rh.handlerName)) defer func() { timer.ObserveDuration() }() + rh.recordRequest(r, w.LocalAddr(), w.RemoteAddr()) + m := new(dns.Msg) m.Compress = false m.SetReply(r) - switch r.Opcode { - case dns.OpcodeQuery: + if r.Opcode == dns.OpcodeQuery { rh.handleQuery(m) } if err := w.WriteMsg(m); err != nil { @@ -40,35 +46,32 @@ func (rh regexHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { } } -func (rh regexHandler) handleQuery(m *dns.Msg) { +func (rh *regexHandler) handleQuery(m *dns.Msg) { for _, q := range m.Question { - rh.logger.Info( - "handling question", - zap.String("question", q.Name), - ) switch q.Qtype { case dns.TypeA: totalHandledRequestsCounter.WithLabelValues(rh.handlerName).Inc() for _, rule := range rh.routes { - if rule.pattern.MatchString(q.Name) { - m.Authoritative = true - answer := &dns.A{ - Hdr: dns.RR_Header{ - Name: q.Name, - Rrtype: dns.TypeA, - Class: dns.ClassINET, - Ttl: 60, - }, - A: rule.response, - } - m.Answer = append(m.Answer, answer) - rh.logger.Info( - "matched DNS rule", - zap.String("pattern", rule.pattern.String()), - zap.String("response", rule.response.String()), - ) - return + if !rule.pattern.MatchString(q.Name) { + continue } + m.Authoritative = true + answer := &dns.A{ + Hdr: dns.RR_Header{ + Name: q.Name, + Rrtype: dns.TypeA, + Class: dns.ClassINET, + Ttl: 60, + }, + A: rule.response, + } + m.Answer = append(m.Answer, answer) + rh.logger.Info( + "matched DNS rule", + zap.String("pattern", rule.pattern.String()), + zap.String("response", rule.response.String()), + ) + return } rh.handleFallbackForMessage(m, q) default: @@ -81,7 +84,7 @@ func (rh regexHandler) handleQuery(m *dns.Msg) { } } -func (rh regexHandler) handleFallbackForMessage(m *dns.Msg, q dns.Question) { +func (rh *regexHandler) handleFallbackForMessage(m *dns.Msg, q dns.Question) { fallbackIP := rh.fallback.GetIP() answer := &dns.A{ Hdr: dns.RR_Header{ @@ -99,3 +102,38 @@ func (rh regexHandler) handleFallbackForMessage(m *dns.Msg, q dns.Question) { m.Authoritative = true m.Answer = append(m.Answer, answer) } + +func (rh *regexHandler) recordRequest(m *dns.Msg, localAddr, remoteAddr net.Addr) { + dnsDetails := &details.DNS{ + OPCode: details.DNSOpCode(m.Opcode), + } + + for _, q := range m.Question { + dnsDetails.Questions = append(dnsDetails.Questions, details.DNSQuestion{ + RRType: details.ResourceRecordType(q.Qtype), + Name: q.Name, + }) + } + + ev := audit.Event{ + Transport: guessTransportFromAddr(localAddr), + Application: audit.AppProtocol_DNS, + ProtocolDetails: dnsDetails, + } + + ev.SetSourceIPFromAddr(remoteAddr) + ev.SetDestinationIPFromAddr(localAddr) + + rh.auditEmitter.Emit(ev) +} + +func guessTransportFromAddr(addr net.Addr) audit.TransportProtocol { + switch addr.(type) { + case *net.TCPAddr: + return audit.TransportProtocol_TCP + case *net.UDPAddr: + return audit.TransportProtocol_UDP + default: + return audit.TransportProtocol_UNKNOWN_TRANSPORT + } +} diff --git a/plugins/dns_mock/register.go b/plugins/dns_mock/register.go index ed32569..feedf31 100644 --- a/plugins/dns_mock/register.go +++ b/plugins/dns_mock/register.go @@ -3,9 +3,7 @@ package dns_mock import ( "github.com/prometheus/client_golang/prometheus" "gitlab.com/inetmock/inetmock/pkg/api" - "gitlab.com/inetmock/inetmock/pkg/logging" "gitlab.com/inetmock/inetmock/pkg/metrics" - "go.uber.org/zap" ) const ( @@ -20,14 +18,6 @@ var ( ) func AddDNSMock(registry api.HandlerRegistry) (err error) { - var logger logging.Logger - if logger, err = logging.CreateLogger(); err != nil { - return - } - logger = logger.With( - zap.String("protocol_handler", name), - ) - if totalHandledRequestsCounter, err = metrics.Counter( name, "handled_requests_total", @@ -57,9 +47,7 @@ func AddDNSMock(registry api.HandlerRegistry) (err error) { } registry.RegisterHandler(name, func() api.ProtocolHandler { - return &dnsHandler{ - logger: logger, - } + return &dnsHandler{} }) return diff --git a/plugins/http_mock/protocol_options.go b/plugins/http_mock/protocol_options.go index 600cfb8..a702ee5 100644 --- a/plugins/http_mock/protocol_options.go +++ b/plugins/http_mock/protocol_options.go @@ -82,7 +82,7 @@ func loadFromConfig(config *viper.Viper) (options httpOptions, err error) { } if absoluteResponsePath, parseErr = filepath.Abs(i.Response); parseErr != nil { - + continue } options.Rules = append(options.Rules, targetRule{ diff --git a/plugins/http_mock/regex_router.go b/plugins/http_mock/regex_router.go index 9b643d1..f9af3c9 100644 --- a/plugins/http_mock/regex_router.go +++ b/plugins/http_mock/regex_router.go @@ -1,16 +1,14 @@ package http_mock import ( - "net" "net/http" "strconv" - "strings" "github.com/prometheus/client_golang/prometheus" "gitlab.com/inetmock/inetmock/pkg/audit" + details "gitlab.com/inetmock/inetmock/pkg/audit/details" "gitlab.com/inetmock/inetmock/pkg/logging" "go.uber.org/zap" - "google.golang.org/protobuf/types/known/anypb" ) type route struct { @@ -70,47 +68,22 @@ func (f emittingFileHandler) ServeHTTP(writer http.ResponseWriter, request *http } func eventFromRequest(request *http.Request) audit.Event { - details := audit.HTTPDetails{ + httpDetails := details.HTTP{ Method: request.Method, Host: request.Host, URI: request.RequestURI, Proto: request.Proto, Headers: request.Header, } - var wire *anypb.Any - if w, err := details.MarshalToWireFormat(); err == nil { - wire = w - } - localIP, localPort := ipPortFromAddr(LocalAddr(request.Context())) - remoteIP, remotePort := ipPortFromAddr(RemoteAddr(request.Context())) - - return audit.Event{ + ev := audit.Event{ Transport: audit.TransportProtocol_TCP, Application: audit.AppProtocol_HTTP, - SourceIP: remoteIP, - DestinationIP: localIP, - SourcePort: remotePort, - DestinationPort: localPort, - ProtocolDetails: wire, + ProtocolDetails: httpDetails, } -} - -func ipPortFromAddr(addr net.Addr) (ip net.IP, port uint16) { - if tcpAddr, isTCPAddr := addr.(*net.TCPAddr); isTCPAddr { - return tcpAddr.IP, uint16(tcpAddr.Port) - } - - ipPortSplit := strings.Split(addr.String(), ":") - if len(ipPortSplit) != 2 { - return - } - - ip = net.ParseIP(ipPortSplit[0]) - - if p, err := strconv.Atoi(ipPortSplit[1]); err == nil { - port = uint16(p) - } - - return + + ev.SetDestinationIPFromAddr(LocalAddr(request.Context())) + ev.SetSourceIPFromAddr(RemoteAddr(request.Context())) + + return ev } From bb9f45ce9177372bd0beaf3a64b28075dd63356a Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Wed, 13 Jan 2021 17:59:08 +0100 Subject: [PATCH 06/26] Move sinks to extra package --- internal/app/app.go | 3 +- pkg/audit/event_stream_test.go | 16 ++------ pkg/audit/{ => sink}/log_sink.go | 9 +++-- pkg/audit/{ => sink}/log_sink_test.go | 8 ++-- pkg/audit/{ => sink}/writer_sink.go | 16 ++++---- pkg/audit/{ => sink}/writer_sink_test.go | 47 ++++++++++++++++++++++-- pkg/audit/writer.go | 4 ++ pkg/wait/wg.go | 14 +++++++ 8 files changed, 84 insertions(+), 33 deletions(-) rename pkg/audit/{ => sink}/log_sink.go (82%) rename pkg/audit/{ => sink}/log_sink_test.go (91%) rename pkg/audit/{ => sink}/writer_sink.go (63%) rename pkg/audit/{ => sink}/writer_sink_test.go (53%) create mode 100644 pkg/wait/wg.go diff --git a/internal/app/app.go b/internal/app/app.go index 6c4adfe..2b17ec1 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -12,6 +12,7 @@ import ( "gitlab.com/inetmock/inetmock/internal/endpoints" "gitlab.com/inetmock/inetmock/pkg/api" "gitlab.com/inetmock/inetmock/pkg/audit" + "gitlab.com/inetmock/inetmock/pkg/audit/sink" "gitlab.com/inetmock/inetmock/pkg/cert" "gitlab.com/inetmock/inetmock/pkg/config" "gitlab.com/inetmock/inetmock/pkg/health" @@ -168,7 +169,7 @@ func NewApp(registrations ...api.Registration) (inetmockApp App, err error) { return } - err = a.eventStream.RegisterSink(audit.NewLogSink(a.Logger().Named("LogSink"))) + err = a.eventStream.RegisterSink(sink.NewLogSink(a.Logger().Named("LogSink"))) return } diff --git a/pkg/audit/event_stream_test.go b/pkg/audit/event_stream_test.go index 8e25eca..1d47fdc 100644 --- a/pkg/audit/event_stream_test.go +++ b/pkg/audit/event_stream_test.go @@ -11,6 +11,7 @@ import ( "gitlab.com/inetmock/inetmock/pkg/audit" "gitlab.com/inetmock/inetmock/pkg/audit/details" "gitlab.com/inetmock/inetmock/pkg/logging" + "gitlab.com/inetmock/inetmock/pkg/wait" ) var ( @@ -215,13 +216,13 @@ func Test_eventStream_Emit(t *testing.T) { }(tt.args.evs, emittedWaitGroup) select { - case <-waitGroupDone(emittedWaitGroup): + case <-wait.ForWaitGroupDone(emittedWaitGroup): case <-time.After(100 * time.Millisecond): t.Errorf("not all events emitted in time") } select { - case <-waitGroupDone(receivedWaitGroup): + case <-wait.ForWaitGroupDone(receivedWaitGroup): case <-time.After(5 * time.Second): t.Errorf("did not get all expected events in time") } @@ -232,14 +233,3 @@ func Test_eventStream_Emit(t *testing.T) { t.Run(tt.name, scenario(tt)) } } - -func waitGroupDone(wg *sync.WaitGroup) <-chan struct{} { - done := make(chan struct{}) - - go func(wg *sync.WaitGroup) { - wg.Wait() - close(done) - }(wg) - - return done -} diff --git a/pkg/audit/log_sink.go b/pkg/audit/sink/log_sink.go similarity index 82% rename from pkg/audit/log_sink.go rename to pkg/audit/sink/log_sink.go index a6e07f1..9b9de78 100644 --- a/pkg/audit/log_sink.go +++ b/pkg/audit/sink/log_sink.go @@ -1,8 +1,9 @@ -package audit +package sink import ( "crypto/tls" + "gitlab.com/inetmock/inetmock/pkg/audit" "gitlab.com/inetmock/inetmock/pkg/logging" "go.uber.org/zap" ) @@ -11,7 +12,7 @@ const ( logSinkName = "logging" ) -func NewLogSink(logger logging.Logger) Sink { +func NewLogSink(logger logging.Logger) audit.Sink { return &logSink{ logger: logger, } @@ -25,8 +26,8 @@ func (logSink) Name() string { return logSinkName } -func (l logSink) OnSubscribe(evs <-chan Event) { - go func(logger logging.Logger, evs <-chan Event) { +func (l logSink) OnSubscribe(evs <-chan audit.Event) { + go func(logger logging.Logger, evs <-chan audit.Event) { for ev := range evs { eventLogger := logger diff --git a/pkg/audit/log_sink_test.go b/pkg/audit/sink/log_sink_test.go similarity index 91% rename from pkg/audit/log_sink_test.go rename to pkg/audit/sink/log_sink_test.go index f201aa5..c64f200 100644 --- a/pkg/audit/log_sink_test.go +++ b/pkg/audit/sink/log_sink_test.go @@ -1,4 +1,4 @@ -package audit_test +package sink_test import ( "sync" @@ -8,7 +8,9 @@ import ( "github.com/golang/mock/gomock" logging_mock "gitlab.com/inetmock/inetmock/internal/mock/logging" "gitlab.com/inetmock/inetmock/pkg/audit" + "gitlab.com/inetmock/inetmock/pkg/audit/sink" "gitlab.com/inetmock/inetmock/pkg/logging" + "gitlab.com/inetmock/inetmock/pkg/wait" "go.uber.org/zap" ) @@ -81,7 +83,7 @@ func Test_logSink_OnSubscribe(t *testing.T) { wg := new(sync.WaitGroup) wg.Add(len(tt.events)) - logSink := audit.NewLogSink(tt.fields.loggerSetup(t, wg)) + logSink := sink.NewLogSink(tt.fields.loggerSetup(t, wg)) var evs audit.EventStream var err error if evs, err = audit.NewEventStream(logging.CreateTestLogger(t)); err != nil { @@ -99,7 +101,7 @@ func Test_logSink_OnSubscribe(t *testing.T) { select { case <-time.After(100 * time.Millisecond): t.Errorf("not all events recorded in time") - case <-waitGroupDone(wg): + case <-wait.ForWaitGroupDone(wg): } } } diff --git a/pkg/audit/writer_sink.go b/pkg/audit/sink/writer_sink.go similarity index 63% rename from pkg/audit/writer_sink.go rename to pkg/audit/sink/writer_sink.go index 803d29a..e0e2ba5 100644 --- a/pkg/audit/writer_sink.go +++ b/pkg/audit/sink/writer_sink.go @@ -1,4 +1,6 @@ -package audit +package sink + +import "gitlab.com/inetmock/inetmock/pkg/audit" type WriterSinkOption func(sink *writerCloserSink) @@ -8,7 +10,7 @@ var ( } ) -func NewWriterSink(name string, target Writer, opts ...WriterSinkOption) Sink { +func NewWriterSink(name string, target audit.Writer, opts ...WriterSinkOption) audit.Sink { sink := &writerCloserSink{ name: name, target: target, @@ -23,20 +25,16 @@ func NewWriterSink(name string, target Writer, opts ...WriterSinkOption) Sink { type writerCloserSink struct { name string - target Writer + target audit.Writer closeOnExit bool } -type syncer interface { - Sync() error -} - func (f writerCloserSink) Name() string { return f.name } -func (f writerCloserSink) OnSubscribe(evs <-chan Event) { - go func(target Writer, closeOnExit bool, evs <-chan Event) { +func (f writerCloserSink) OnSubscribe(evs <-chan audit.Event) { + go func(target audit.Writer, closeOnExit bool, evs <-chan audit.Event) { for ev := range evs { _ = target.Write(&ev) } diff --git a/pkg/audit/writer_sink_test.go b/pkg/audit/sink/writer_sink_test.go similarity index 53% rename from pkg/audit/writer_sink_test.go rename to pkg/audit/sink/writer_sink_test.go index 483a5ac..0791369 100644 --- a/pkg/audit/writer_sink_test.go +++ b/pkg/audit/sink/writer_sink_test.go @@ -1,6 +1,9 @@ -package audit_test +package sink_test import ( + "crypto/tls" + "net" + "net/http" "sync" "testing" "time" @@ -8,7 +11,45 @@ import ( "github.com/golang/mock/gomock" audit_mock "gitlab.com/inetmock/inetmock/internal/mock/audit" "gitlab.com/inetmock/inetmock/pkg/audit" + "gitlab.com/inetmock/inetmock/pkg/audit/details" + "gitlab.com/inetmock/inetmock/pkg/audit/sink" "gitlab.com/inetmock/inetmock/pkg/logging" + "gitlab.com/inetmock/inetmock/pkg/wait" +) + +var ( + testEvents = []*audit.Event{ + { + Transport: audit.TransportProtocol_TCP, + Application: audit.AppProtocol_HTTP, + SourceIP: net.ParseIP("127.0.0.1").To4(), + DestinationIP: net.ParseIP("127.0.0.1").To4(), + SourcePort: 32344, + DestinationPort: 80, + TLS: &audit.TLSDetails{ + Version: tls.VersionTLS13, + CipherSuite: tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + ServerName: "localhost", + }, + ProtocolDetails: details.HTTP{ + Method: "GET", + Host: "localhost", + URI: "http://localhost/asdf", + Proto: "HTTP 1.1", + Headers: http.Header{ + "Accept": []string{"application/json"}, + }, + }, + }, + { + Transport: audit.TransportProtocol_TCP, + Application: audit.AppProtocol_DNS, + SourceIP: net.ParseIP("::1").To16(), + DestinationIP: net.ParseIP("::1").To16(), + SourcePort: 32344, + DestinationPort: 80, + }, + } ) func Test_writerCloserSink_OnSubscribe(t *testing.T) { @@ -42,7 +83,7 @@ func Test_writerCloserSink_OnSubscribe(t *testing.T) { }). Times(len(tt.events)) - writerCloserSink := audit.NewWriterSink("WriterMock", writerMock, audit.WithCloseOnExit) + writerCloserSink := sink.NewWriterSink("WriterMock", writerMock, sink.WithCloseOnExit) var evs audit.EventStream var err error @@ -61,7 +102,7 @@ func Test_writerCloserSink_OnSubscribe(t *testing.T) { select { case <-time.After(100 * time.Millisecond): t.Errorf("not all events recorded in time") - case <-waitGroupDone(wg): + case <-wait.ForWaitGroupDone(wg): } } } diff --git a/pkg/audit/writer.go b/pkg/audit/writer.go index 5a318b4..15424d7 100644 --- a/pkg/audit/writer.go +++ b/pkg/audit/writer.go @@ -52,6 +52,10 @@ type eventWriter struct { byteOrder binary.ByteOrder } +type syncer interface { + Sync() error +} + func (e eventWriter) Write(ev *Event) (err error) { if ev == nil { return ErrValueMostNotBeNil diff --git a/pkg/wait/wg.go b/pkg/wait/wg.go new file mode 100644 index 0000000..11adf3f --- /dev/null +++ b/pkg/wait/wg.go @@ -0,0 +1,14 @@ +package wait + +import "sync" + +func ForWaitGroupDone(wg *sync.WaitGroup) <-chan struct{} { + done := make(chan struct{}) + + go func(wg *sync.WaitGroup) { + wg.Wait() + close(done) + }(wg) + + return done +} From dc4a9b18a328df78b1c30dde76a6d444f261c53c Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Wed, 13 Jan 2021 18:07:04 +0100 Subject: [PATCH 07/26] Moved endpoint handlers in tree --- cmd/inetmock/main.go | 10 +++++----- internal/app/app.go | 10 +++++----- internal/cmd/server.go | 20 +++++++++---------- internal/{endpoints => endpoint}/constants.go | 2 +- internal/{endpoints => endpoint}/endpoint.go | 2 +- .../endpoint_manager.go | 2 +- .../endpoint_manager_test.go | 2 +- .../endpoint/handler/dns/mock}/fallback.go | 2 +- .../handler/dns/mock}/fallback_test.go | 2 +- .../endpoint/handler/dns/mock}/handler.go | 2 +- .../handler/dns/mock}/protocol_options.go | 2 +- .../handler/dns/mock}/regex_handler.go | 2 +- .../endpoint/handler/dns/mock}/register.go | 2 +- .../handler/http/mock}/conn_context.go | 2 +- .../endpoint/handler/http/mock}/handler.go | 2 +- .../handler/http/mock}/handler_test.go | 6 +++--- .../handler/http/mock}/protocol_options.go | 2 +- .../http/mock}/protocol_options_test.go | 2 +- .../handler/http/mock}/regex_router.go | 2 +- .../endpoint/handler/http/mock}/register.go | 2 +- .../endpoint/handler/http/proxy}/handler.go | 2 +- .../handler/http/proxy}/protocol_options.go | 2 +- .../handler/http/proxy}/proxy_handler.go | 2 +- .../endpoint/handler/http/proxy}/register.go | 2 +- .../endpoint/handler/metrics}/handler.go | 2 +- .../handler/metrics}/protocol_options.go | 2 +- .../endpoint/handler/metrics}/register.go | 2 +- .../handler/tls/interceptor}/handler.go | 2 +- .../tls/interceptor}/protocol_options.go | 2 +- .../handler/tls/interceptor}/proxy.go | 2 +- .../handler/tls/interceptor}/proxy_conn.go | 2 +- .../handler/tls/interceptor}/register.go | 2 +- internal/rpc/endpoints_server.go | 8 ++++---- 33 files changed, 55 insertions(+), 55 deletions(-) rename internal/{endpoints => endpoint}/constants.go (77%) rename internal/{endpoints => endpoint}/endpoint.go (98%) rename internal/{endpoints => endpoint}/endpoint_manager.go (99%) rename internal/{endpoints => endpoint}/endpoint_manager_test.go (99%) rename {plugins/dns_mock => internal/endpoint/handler/dns/mock}/fallback.go (98%) rename {plugins/dns_mock => internal/endpoint/handler/dns/mock}/fallback_test.go (99%) rename {plugins/dns_mock => internal/endpoint/handler/dns/mock}/handler.go (99%) rename {plugins/dns_mock => internal/endpoint/handler/dns/mock}/protocol_options.go (98%) rename {plugins/dns_mock => internal/endpoint/handler/dns/mock}/regex_handler.go (99%) rename {plugins/dns_mock => internal/endpoint/handler/dns/mock}/register.go (98%) rename {plugins/http_mock => internal/endpoint/handler/http/mock}/conn_context.go (97%) rename {plugins/http_mock => internal/endpoint/handler/http/mock}/handler.go (98%) rename {plugins/http_mock => internal/endpoint/handler/http/mock}/handler_test.go (95%) rename {plugins/http_mock => internal/endpoint/handler/http/mock}/protocol_options.go (99%) rename {plugins/http_mock => internal/endpoint/handler/http/mock}/protocol_options_test.go (99%) rename {plugins/http_mock => internal/endpoint/handler/http/mock}/regex_router.go (99%) rename {plugins/http_mock => internal/endpoint/handler/http/mock}/register.go (97%) rename {plugins/http_proxy => internal/endpoint/handler/http/proxy}/handler.go (98%) rename {plugins/http_proxy => internal/endpoint/handler/http/proxy}/protocol_options.go (92%) rename {plugins/http_proxy => internal/endpoint/handler/http/proxy}/proxy_handler.go (99%) rename {plugins/http_proxy => internal/endpoint/handler/http/proxy}/register.go (98%) rename {plugins/metrics_exporter => internal/endpoint/handler/metrics}/handler.go (97%) rename {plugins/metrics_exporter => internal/endpoint/handler/metrics}/protocol_options.go (68%) rename {plugins/metrics_exporter => internal/endpoint/handler/metrics}/register.go (94%) rename {plugins/tls_interceptor => internal/endpoint/handler/tls/interceptor}/handler.go (99%) rename {plugins/tls_interceptor => internal/endpoint/handler/tls/interceptor}/protocol_options.go (91%) rename {plugins/tls_interceptor => internal/endpoint/handler/tls/interceptor}/proxy.go (97%) rename {plugins/tls_interceptor => internal/endpoint/handler/tls/interceptor}/proxy_conn.go (94%) rename {plugins/tls_interceptor => internal/endpoint/handler/tls/interceptor}/register.go (98%) diff --git a/cmd/inetmock/main.go b/cmd/inetmock/main.go index f478e04..38cf492 100644 --- a/cmd/inetmock/main.go +++ b/cmd/inetmock/main.go @@ -2,11 +2,11 @@ package main import ( "gitlab.com/inetmock/inetmock/internal/cmd" - _ "gitlab.com/inetmock/inetmock/plugins/dns_mock" - _ "gitlab.com/inetmock/inetmock/plugins/http_mock" - _ "gitlab.com/inetmock/inetmock/plugins/http_proxy" - _ "gitlab.com/inetmock/inetmock/plugins/metrics_exporter" - _ "gitlab.com/inetmock/inetmock/plugins/tls_interceptor" + _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/dns/mock" + _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/mock" + _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/proxy" + _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/metrics" + _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/tls/interceptor" ) func main() { diff --git a/internal/app/app.go b/internal/app/app.go index 2b17ec1..f906795 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -9,7 +9,7 @@ import ( "syscall" "github.com/spf13/cobra" - "gitlab.com/inetmock/inetmock/internal/endpoints" + "gitlab.com/inetmock/inetmock/internal/endpoint" "gitlab.com/inetmock/inetmock/pkg/api" "gitlab.com/inetmock/inetmock/pkg/audit" "gitlab.com/inetmock/inetmock/pkg/audit/sink" @@ -31,7 +31,7 @@ type App interface { api.PluginContext Config() config.Config Checker() health.Checker - EndpointManager() endpoints.EndpointManager + EndpointManager() endpoint.EndpointManager HandlerRegistry() api.HandlerRegistry Context() context.Context MustRun() @@ -45,7 +45,7 @@ type app struct { rootLogger logging.Logger certStore cert.Store checker health.Checker - endpointManager endpoints.EndpointManager + endpointManager endpoint.EndpointManager registry api.HandlerRegistry ctx context.Context cancel context.CancelFunc @@ -81,7 +81,7 @@ func (a app) Checker() health.Checker { return a.checker } -func (a app) EndpointManager() endpoints.EndpointManager { +func (a app) EndpointManager() endpoint.EndpointManager { return a.endpointManager } @@ -144,7 +144,7 @@ func NewApp(registrations ...api.Registration) (inetmockApp App, err error) { return } - a.endpointManager = endpoints.NewEndpointManager( + a.endpointManager = endpoint.NewEndpointManager( a.registry, a.Logger().Named("EndpointManager"), a.checker, diff --git a/internal/cmd/server.go b/internal/cmd/server.go index b182b88..d3d9bc1 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -5,11 +5,11 @@ import ( "os" "gitlab.com/inetmock/inetmock/internal/app" - "gitlab.com/inetmock/inetmock/plugins/dns_mock" - "gitlab.com/inetmock/inetmock/plugins/http_mock" - "gitlab.com/inetmock/inetmock/plugins/http_proxy" - "gitlab.com/inetmock/inetmock/plugins/metrics_exporter" - "gitlab.com/inetmock/inetmock/plugins/tls_interceptor" + "gitlab.com/inetmock/inetmock/internal/endpoint/handler/dns/mock" + mock2 "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/mock" + "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/proxy" + "gitlab.com/inetmock/inetmock/internal/endpoint/handler/metrics" + "gitlab.com/inetmock/inetmock/internal/endpoint/handler/tls/interceptor" ) var ( @@ -19,11 +19,11 @@ var ( func ExecuteServerCommand() { var err error if server, err = app.NewApp( - http_mock.AddHTTPMock, - dns_mock.AddDNSMock, - tls_interceptor.AddTLSInterceptor, - http_proxy.AddHTTPProxy, - metrics_exporter.AddMetricsExporter, + mock2.AddHTTPMock, + mock.AddDNSMock, + interceptor.AddTLSInterceptor, + proxy.AddHTTPProxy, + metrics.AddMetricsExporter, ); err != nil { fmt.Println(err.Error()) os.Exit(1) diff --git a/internal/endpoints/constants.go b/internal/endpoint/constants.go similarity index 77% rename from internal/endpoints/constants.go rename to internal/endpoint/constants.go index 1e9f681..b7379eb 100644 --- a/internal/endpoints/constants.go +++ b/internal/endpoint/constants.go @@ -1,4 +1,4 @@ -package endpoints +package endpoint import "time" diff --git a/internal/endpoints/endpoint.go b/internal/endpoint/endpoint.go similarity index 98% rename from internal/endpoints/endpoint.go rename to internal/endpoint/endpoint.go index f27068b..9dd314a 100644 --- a/internal/endpoints/endpoint.go +++ b/internal/endpoint/endpoint.go @@ -1,5 +1,5 @@ //go:generate mockgen -source=$GOFILE -destination=./../../internal/mock/endpoints/endpoint.mock.go -package=endpoints_mock -package endpoints +package endpoint import ( "context" diff --git a/internal/endpoints/endpoint_manager.go b/internal/endpoint/endpoint_manager.go similarity index 99% rename from internal/endpoints/endpoint_manager.go rename to internal/endpoint/endpoint_manager.go index 0470f8d..2da4be2 100644 --- a/internal/endpoints/endpoint_manager.go +++ b/internal/endpoint/endpoint_manager.go @@ -1,4 +1,4 @@ -package endpoints +package endpoint import ( "context" diff --git a/internal/endpoints/endpoint_manager_test.go b/internal/endpoint/endpoint_manager_test.go similarity index 99% rename from internal/endpoints/endpoint_manager_test.go rename to internal/endpoint/endpoint_manager_test.go index 0e49092..0c71cf4 100644 --- a/internal/endpoints/endpoint_manager_test.go +++ b/internal/endpoint/endpoint_manager_test.go @@ -1,4 +1,4 @@ -package endpoints +package endpoint import ( "reflect" diff --git a/plugins/dns_mock/fallback.go b/internal/endpoint/handler/dns/mock/fallback.go similarity index 98% rename from plugins/dns_mock/fallback.go rename to internal/endpoint/handler/dns/mock/fallback.go index 43697b1..8fb39a8 100644 --- a/plugins/dns_mock/fallback.go +++ b/internal/endpoint/handler/dns/mock/fallback.go @@ -1,4 +1,4 @@ -package dns_mock +package mock import ( "encoding/binary" diff --git a/plugins/dns_mock/fallback_test.go b/internal/endpoint/handler/dns/mock/fallback_test.go similarity index 99% rename from plugins/dns_mock/fallback_test.go rename to internal/endpoint/handler/dns/mock/fallback_test.go index a1996b3..33622be 100644 --- a/plugins/dns_mock/fallback_test.go +++ b/internal/endpoint/handler/dns/mock/fallback_test.go @@ -1,4 +1,4 @@ -package dns_mock +package mock import ( "net" diff --git a/plugins/dns_mock/handler.go b/internal/endpoint/handler/dns/mock/handler.go similarity index 99% rename from plugins/dns_mock/handler.go rename to internal/endpoint/handler/dns/mock/handler.go index 0f9d372..5e05432 100644 --- a/plugins/dns_mock/handler.go +++ b/internal/endpoint/handler/dns/mock/handler.go @@ -1,4 +1,4 @@ -package dns_mock +package mock import ( "context" diff --git a/plugins/dns_mock/protocol_options.go b/internal/endpoint/handler/dns/mock/protocol_options.go similarity index 98% rename from plugins/dns_mock/protocol_options.go rename to internal/endpoint/handler/dns/mock/protocol_options.go index f3aabc6..ea74351 100644 --- a/plugins/dns_mock/protocol_options.go +++ b/internal/endpoint/handler/dns/mock/protocol_options.go @@ -1,4 +1,4 @@ -package dns_mock +package mock import ( "net" diff --git a/plugins/dns_mock/regex_handler.go b/internal/endpoint/handler/dns/mock/regex_handler.go similarity index 99% rename from plugins/dns_mock/regex_handler.go rename to internal/endpoint/handler/dns/mock/regex_handler.go index 80dcb6a..64905b0 100644 --- a/plugins/dns_mock/regex_handler.go +++ b/internal/endpoint/handler/dns/mock/regex_handler.go @@ -1,4 +1,4 @@ -package dns_mock +package mock import ( "net" diff --git a/plugins/dns_mock/register.go b/internal/endpoint/handler/dns/mock/register.go similarity index 98% rename from plugins/dns_mock/register.go rename to internal/endpoint/handler/dns/mock/register.go index feedf31..312f3e6 100644 --- a/plugins/dns_mock/register.go +++ b/internal/endpoint/handler/dns/mock/register.go @@ -1,4 +1,4 @@ -package dns_mock +package mock import ( "github.com/prometheus/client_golang/prometheus" diff --git a/plugins/http_mock/conn_context.go b/internal/endpoint/handler/http/mock/conn_context.go similarity index 97% rename from plugins/http_mock/conn_context.go rename to internal/endpoint/handler/http/mock/conn_context.go index b22af86..502d104 100644 --- a/plugins/http_mock/conn_context.go +++ b/internal/endpoint/handler/http/mock/conn_context.go @@ -1,4 +1,4 @@ -package http_mock +package mock import ( "context" diff --git a/plugins/http_mock/handler.go b/internal/endpoint/handler/http/mock/handler.go similarity index 98% rename from plugins/http_mock/handler.go rename to internal/endpoint/handler/http/mock/handler.go index f1acb23..486b22e 100644 --- a/plugins/http_mock/handler.go +++ b/internal/endpoint/handler/http/mock/handler.go @@ -1,4 +1,4 @@ -package http_mock +package mock import ( "context" diff --git a/plugins/http_mock/handler_test.go b/internal/endpoint/handler/http/mock/handler_test.go similarity index 95% rename from plugins/http_mock/handler_test.go rename to internal/endpoint/handler/http/mock/handler_test.go index 804943a..e23bbf7 100644 --- a/plugins/http_mock/handler_test.go +++ b/internal/endpoint/handler/http/mock/handler_test.go @@ -1,4 +1,4 @@ -package http_mock_test +package mock_test import ( "context" @@ -13,12 +13,12 @@ import ( "github.com/golang/mock/gomock" "github.com/spf13/viper" + "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/mock" api_mock "gitlab.com/inetmock/inetmock/internal/mock/api" audit_mock "gitlab.com/inetmock/inetmock/internal/mock/audit" "gitlab.com/inetmock/inetmock/pkg/api" "gitlab.com/inetmock/inetmock/pkg/config" "gitlab.com/inetmock/inetmock/pkg/logging" - "gitlab.com/inetmock/inetmock/plugins/http_mock" "go.uber.org/zap" ) @@ -65,7 +65,7 @@ func setupHandler(b *testing.B, ctrl *gomock.Controller, listenPort uint16) (api b.Helper() registry := api.NewHandlerRegistry() - if err := http_mock.AddHTTPMock(registry); err != nil { + if err := mock.AddHTTPMock(registry); err != nil { b.Errorf("AddHTTPMock() error = %v", err) } handler, ok := registry.HandlerForName("http_mock") diff --git a/plugins/http_mock/protocol_options.go b/internal/endpoint/handler/http/mock/protocol_options.go similarity index 99% rename from plugins/http_mock/protocol_options.go rename to internal/endpoint/handler/http/mock/protocol_options.go index a702ee5..cca7a4d 100644 --- a/plugins/http_mock/protocol_options.go +++ b/internal/endpoint/handler/http/mock/protocol_options.go @@ -1,5 +1,5 @@ //go:generate go-enum -f $GOFILE --lower --marshal --names -package http_mock +package mock import ( "net/http" diff --git a/plugins/http_mock/protocol_options_test.go b/internal/endpoint/handler/http/mock/protocol_options_test.go similarity index 99% rename from plugins/http_mock/protocol_options_test.go rename to internal/endpoint/handler/http/mock/protocol_options_test.go index e992c2f..e56bc7e 100644 --- a/plugins/http_mock/protocol_options_test.go +++ b/internal/endpoint/handler/http/mock/protocol_options_test.go @@ -1,4 +1,4 @@ -package http_mock +package mock import ( "path/filepath" diff --git a/plugins/http_mock/regex_router.go b/internal/endpoint/handler/http/mock/regex_router.go similarity index 99% rename from plugins/http_mock/regex_router.go rename to internal/endpoint/handler/http/mock/regex_router.go index f9af3c9..4e96329 100644 --- a/plugins/http_mock/regex_router.go +++ b/internal/endpoint/handler/http/mock/regex_router.go @@ -1,4 +1,4 @@ -package http_mock +package mock import ( "net/http" diff --git a/plugins/http_mock/register.go b/internal/endpoint/handler/http/mock/register.go similarity index 97% rename from plugins/http_mock/register.go rename to internal/endpoint/handler/http/mock/register.go index d81e055..b84738f 100644 --- a/plugins/http_mock/register.go +++ b/internal/endpoint/handler/http/mock/register.go @@ -1,4 +1,4 @@ -package http_mock +package mock import ( "github.com/prometheus/client_golang/prometheus" diff --git a/plugins/http_proxy/handler.go b/internal/endpoint/handler/http/proxy/handler.go similarity index 98% rename from plugins/http_proxy/handler.go rename to internal/endpoint/handler/http/proxy/handler.go index 251a43e..3eef03a 100644 --- a/plugins/http_proxy/handler.go +++ b/internal/endpoint/handler/http/proxy/handler.go @@ -1,4 +1,4 @@ -package http_proxy +package proxy import ( "context" diff --git a/plugins/http_proxy/protocol_options.go b/internal/endpoint/handler/http/proxy/protocol_options.go similarity index 92% rename from plugins/http_proxy/protocol_options.go rename to internal/endpoint/handler/http/proxy/protocol_options.go index 5ec23eb..3dbf34c 100644 --- a/plugins/http_proxy/protocol_options.go +++ b/internal/endpoint/handler/http/proxy/protocol_options.go @@ -1,4 +1,4 @@ -package http_proxy +package proxy import ( "fmt" diff --git a/plugins/http_proxy/proxy_handler.go b/internal/endpoint/handler/http/proxy/proxy_handler.go similarity index 99% rename from plugins/http_proxy/proxy_handler.go rename to internal/endpoint/handler/http/proxy/proxy_handler.go index 9bf6ccb..d7af5b8 100644 --- a/plugins/http_proxy/proxy_handler.go +++ b/internal/endpoint/handler/http/proxy/proxy_handler.go @@ -1,4 +1,4 @@ -package http_proxy +package proxy import ( "context" diff --git a/plugins/http_proxy/register.go b/internal/endpoint/handler/http/proxy/register.go similarity index 98% rename from plugins/http_proxy/register.go rename to internal/endpoint/handler/http/proxy/register.go index cd743e5..4470caf 100644 --- a/plugins/http_proxy/register.go +++ b/internal/endpoint/handler/http/proxy/register.go @@ -1,4 +1,4 @@ -package http_proxy +package proxy import ( "github.com/prometheus/client_golang/prometheus" diff --git a/plugins/metrics_exporter/handler.go b/internal/endpoint/handler/metrics/handler.go similarity index 97% rename from plugins/metrics_exporter/handler.go rename to internal/endpoint/handler/metrics/handler.go index 31aaac5..0519892 100644 --- a/plugins/metrics_exporter/handler.go +++ b/internal/endpoint/handler/metrics/handler.go @@ -1,4 +1,4 @@ -package metrics_exporter +package metrics import ( "context" diff --git a/plugins/metrics_exporter/protocol_options.go b/internal/endpoint/handler/metrics/protocol_options.go similarity index 68% rename from plugins/metrics_exporter/protocol_options.go rename to internal/endpoint/handler/metrics/protocol_options.go index 89de17c..b7b1aed 100644 --- a/plugins/metrics_exporter/protocol_options.go +++ b/internal/endpoint/handler/metrics/protocol_options.go @@ -1,4 +1,4 @@ -package metrics_exporter +package metrics type metricsExporterOptions struct { Route string diff --git a/plugins/metrics_exporter/register.go b/internal/endpoint/handler/metrics/register.go similarity index 94% rename from plugins/metrics_exporter/register.go rename to internal/endpoint/handler/metrics/register.go index f21fcf8..4fa3043 100644 --- a/plugins/metrics_exporter/register.go +++ b/internal/endpoint/handler/metrics/register.go @@ -1,4 +1,4 @@ -package metrics_exporter +package metrics import ( "gitlab.com/inetmock/inetmock/pkg/api" diff --git a/plugins/tls_interceptor/handler.go b/internal/endpoint/handler/tls/interceptor/handler.go similarity index 99% rename from plugins/tls_interceptor/handler.go rename to internal/endpoint/handler/tls/interceptor/handler.go index f5107a7..b180eb2 100644 --- a/plugins/tls_interceptor/handler.go +++ b/internal/endpoint/handler/tls/interceptor/handler.go @@ -1,4 +1,4 @@ -package tls_interceptor +package interceptor import ( "context" diff --git a/plugins/tls_interceptor/protocol_options.go b/internal/endpoint/handler/tls/interceptor/protocol_options.go similarity index 91% rename from plugins/tls_interceptor/protocol_options.go rename to internal/endpoint/handler/tls/interceptor/protocol_options.go index 097ec28..c0de849 100644 --- a/plugins/tls_interceptor/protocol_options.go +++ b/internal/endpoint/handler/tls/interceptor/protocol_options.go @@ -1,4 +1,4 @@ -package tls_interceptor +package interceptor import ( "fmt" diff --git a/plugins/tls_interceptor/proxy.go b/internal/endpoint/handler/tls/interceptor/proxy.go similarity index 97% rename from plugins/tls_interceptor/proxy.go rename to internal/endpoint/handler/tls/interceptor/proxy.go index 5017044..be5af4c 100644 --- a/plugins/tls_interceptor/proxy.go +++ b/internal/endpoint/handler/tls/interceptor/proxy.go @@ -1,4 +1,4 @@ -package tls_interceptor +package interceptor import ( "net" diff --git a/plugins/tls_interceptor/proxy_conn.go b/internal/endpoint/handler/tls/interceptor/proxy_conn.go similarity index 94% rename from plugins/tls_interceptor/proxy_conn.go rename to internal/endpoint/handler/tls/interceptor/proxy_conn.go index f7529d9..396d80f 100644 --- a/plugins/tls_interceptor/proxy_conn.go +++ b/internal/endpoint/handler/tls/interceptor/proxy_conn.go @@ -1,4 +1,4 @@ -package tls_interceptor +package interceptor import ( "fmt" diff --git a/plugins/tls_interceptor/register.go b/internal/endpoint/handler/tls/interceptor/register.go similarity index 98% rename from plugins/tls_interceptor/register.go rename to internal/endpoint/handler/tls/interceptor/register.go index e8ec9cb..6fcf3fc 100644 --- a/plugins/tls_interceptor/register.go +++ b/internal/endpoint/handler/tls/interceptor/register.go @@ -1,4 +1,4 @@ -package tls_interceptor +package interceptor import ( "sync" diff --git a/internal/rpc/endpoints_server.go b/internal/rpc/endpoints_server.go index 7fada0d..ff7fd7a 100644 --- a/internal/rpc/endpoints_server.go +++ b/internal/rpc/endpoints_server.go @@ -3,12 +3,12 @@ package rpc import ( "context" - "gitlab.com/inetmock/inetmock/internal/endpoints" + "gitlab.com/inetmock/inetmock/internal/endpoint" ) type endpointsServer struct { UnimplementedEndpointsServer - endpointsManager endpoints.EndpointManager + endpointsManager endpoint.EndpointManager } func (e endpointsServer) GetEndpoints(_ context.Context, _ *GetEndpointsRequest) (*GetEndpointsResponse, error) { @@ -18,7 +18,7 @@ func (e endpointsServer) GetEndpoints(_ context.Context, _ *GetEndpointsRequest) }, nil } -func rpcEndpointsFromEndpoints(eps []endpoints.Endpoint) *[]*Endpoint { +func rpcEndpointsFromEndpoints(eps []endpoint.Endpoint) *[]*Endpoint { out := make([]*Endpoint, 0) for _, ep := range eps { out = append(out, rpcEndpointFromEndpoint(ep)) @@ -26,7 +26,7 @@ func rpcEndpointsFromEndpoints(eps []endpoints.Endpoint) *[]*Endpoint { return &out } -func rpcEndpointFromEndpoint(ep endpoints.Endpoint) *Endpoint { +func rpcEndpointFromEndpoint(ep endpoint.Endpoint) *Endpoint { return &Endpoint{ Id: ep.Id().String(), Name: ep.Name(), From af31b1166a2d7dc91cbcd0a3736c93db2f74ff37 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Wed, 13 Jan 2021 21:38:52 +0100 Subject: [PATCH 08/26] Audit API prototype - watch events interactively - pipe events to files - remove file subscriptions --- api/proto/internal/rpc/audit.proto | 36 +++++++++++++++ api/proto/internal/rpc/endpoints.proto | 10 ++--- internal/app/app.go | 5 +++ internal/cmd/audit_file.go | 59 ++++++++++++++++++++++++ internal/cmd/audit_watch.go | 62 ++++++++++++++++++++++++++ internal/cmd/cli.go | 44 +++++++++++++++++- internal/cmd/endpoints.go | 9 ++-- internal/format/table_writer.go | 11 +++++ internal/rpc/audit_server.go | 51 +++++++++++++++++++++ internal/rpc/grpc_api.go | 4 ++ internal/rpc/handlers_server.go | 3 -- internal/rpc/health_server.go | 4 +- pkg/audit/api.go | 5 ++- pkg/audit/event.go | 2 +- pkg/audit/event_stream.go | 20 ++++++++- pkg/audit/event_stream_test.go | 2 +- pkg/audit/sink/grpc_sink.go | 39 ++++++++++++++++ pkg/audit/sink/log_sink.go | 2 +- pkg/audit/sink/writer_sink.go | 2 +- 19 files changed, 348 insertions(+), 22 deletions(-) create mode 100644 api/proto/internal/rpc/audit.proto create mode 100644 internal/cmd/audit_file.go create mode 100644 internal/cmd/audit_watch.go create mode 100644 internal/rpc/audit_server.go create mode 100644 pkg/audit/sink/grpc_sink.go diff --git a/api/proto/internal/rpc/audit.proto b/api/proto/internal/rpc/audit.proto new file mode 100644 index 0000000..aa07869 --- /dev/null +++ b/api/proto/internal/rpc/audit.proto @@ -0,0 +1,36 @@ +syntax = "proto3"; + +option go_package = "gitlab.com/inetmock/inetmock/internal/rpc"; +option java_multiple_files = true; +option java_package = "com.github.baez90.inetmock.rpc"; +option java_outer_classname = "AuditProto"; + +import 'pkg/audit/event_entity.proto'; + +package inetmock.rpc; + +message WatchEventsRequest { + string watcherName = 1; +} + +message RegisterFileSinkRequest { + string targetPath = 1; +} + +message RegisterFileSinkResponse { + +} + +message RemoveFileSinkRequest { + string targetPath = 1; +} + +message RemoveFileSinkResponse { + +} + +service Audit { + rpc WatchEvents (WatchEventsRequest) returns (stream inetmock.audit.EventEntity); + rpc RegisterFileSink (RegisterFileSinkRequest) returns (RegisterFileSinkResponse); + rpc RemoveFileSink (RemoveFileSinkRequest) returns (RemoveFileSinkResponse); +} \ No newline at end of file diff --git a/api/proto/internal/rpc/endpoints.proto b/api/proto/internal/rpc/endpoints.proto index 48d850b..db21768 100644 --- a/api/proto/internal/rpc/endpoints.proto +++ b/api/proto/internal/rpc/endpoints.proto @@ -7,11 +7,6 @@ option java_outer_classname = "EndpointsProto"; package inetmock.rpc; -service Endpoints { - rpc GetEndpoints (GetEndpointsRequest) returns (GetEndpointsResponse) { - } -} - message GetEndpointsRequest { } @@ -25,4 +20,9 @@ message Endpoint { string handler = 3; string listenAddress = 4; int32 port = 5; +} + +service Endpoints { + rpc GetEndpoints (GetEndpointsRequest) returns (GetEndpointsResponse) { + } } \ No newline at end of file diff --git a/internal/app/app.go b/internal/app/app.go index f906795..3540e6c 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -29,6 +29,7 @@ var ( type App interface { api.PluginContext + EventStream() audit.EventStream Config() config.Config Checker() health.Checker EndpointManager() endpoint.EndpointManager @@ -89,6 +90,10 @@ func (a app) Audit() audit.Emitter { return a.eventStream } +func (a app) EventStream() audit.EventStream { + return a.eventStream +} + func (a app) HandlerRegistry() api.HandlerRegistry { return a.registry } diff --git a/internal/cmd/audit_file.go b/internal/cmd/audit_file.go new file mode 100644 index 0000000..9f98069 --- /dev/null +++ b/internal/cmd/audit_file.go @@ -0,0 +1,59 @@ +package cmd + +import ( + "context" + "fmt" + "os" + + "github.com/spf13/cobra" + "gitlab.com/inetmock/inetmock/internal/rpc" + "google.golang.org/grpc" +) + +var ( + addFileCmd = &cobra.Command{ + Use: "addFile", + Short: "subscribe events to a file", + Args: cobra.ExactArgs(1), + RunE: runAddFile, + } + + removeFileCmd = &cobra.Command{ + Use: "removeFile", + Short: "remove file subscription", + Args: cobra.ExactArgs(1), + RunE: runRemoveFile, + } +) + +func runAddFile(_ *cobra.Command, args []string) (err error) { + var conn *grpc.ClientConn + + if conn, err = grpc.Dial(inetMockSocketPath, grpc.WithInsecure()); err != nil { + fmt.Printf("Failed to connecto INetMock socket: %v\n", err) + os.Exit(10) + } + + auditClient := rpc.NewAuditClient(conn) + ctx, cancel := context.WithTimeout(appCtx, grpcTimeout) + defer cancel() + + _, err = auditClient.RegisterFileSink(ctx, &rpc.RegisterFileSinkRequest{TargetPath: args[0]}) + return +} + +func runRemoveFile(_ *cobra.Command, args []string) (err error) { + var conn *grpc.ClientConn + + if conn, err = grpc.Dial(inetMockSocketPath, grpc.WithInsecure()); err != nil { + fmt.Printf("Failed to connecto INetMock socket: %v\n", err) + os.Exit(10) + } + + auditClient := rpc.NewAuditClient(conn) + ctx, cancel := context.WithTimeout(appCtx, grpcTimeout) + defer cancel() + + _, err = auditClient.RemoveFileSink(ctx, &rpc.RemoveFileSinkRequest{TargetPath: args[0]}) + return +} diff --git a/internal/cmd/audit_watch.go b/internal/cmd/audit_watch.go new file mode 100644 index 0000000..93e775e --- /dev/null +++ b/internal/cmd/audit_watch.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" + "gitlab.com/inetmock/inetmock/internal/rpc" + "gitlab.com/inetmock/inetmock/pkg/audit" + "google.golang.org/grpc" +) + +var ( + watchEventsCmd = &cobra.Command{ + Use: "watch", + Short: "Watch all audit events", + RunE: watchAuditEvents, + } + + auditCmd = &cobra.Command{ + Use: "audit", + Short: "Interact with the audit stream", + } + + listenerName string +) + +func watchAuditEvents(_ *cobra.Command, _ []string) (err error) { + var conn *grpc.ClientConn + + if conn, err = grpc.Dial(inetMockSocketPath, grpc.WithInsecure()); err != nil { + fmt.Printf("Failed to connecto INetMock socket: %v\n", err) + os.Exit(10) + } + + auditClient := rpc.NewAuditClient(conn) + + var watchClient rpc.Audit_WatchEventsClient + if watchClient, err = auditClient.WatchEvents(appCtx, &rpc.WatchEventsRequest{WatcherName: listenerName}); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + go func() { + var protoEv *audit.EventEntity + for protoEv, err = watchClient.Recv(); err == nil; protoEv, err = watchClient.Recv() { + ev := audit.NewEventFromProto(protoEv) + var out []byte + out, err = json.Marshal(ev) + if err != nil { + continue + } + fmt.Println(string(out)) + } + }() + + <-appCtx.Done() + err = watchClient.CloseSend() + + return +} diff --git a/internal/cmd/cli.go b/internal/cmd/cli.go index 68f5491..bf89ede 100644 --- a/internal/cmd/cli.go +++ b/internal/cmd/cli.go @@ -1,8 +1,14 @@ package cmd import ( + "context" + "os" + "os/signal" + "os/user" + "syscall" "time" + "github.com/google/uuid" "github.com/spf13/cobra" ) @@ -15,20 +21,54 @@ var ( inetMockSocketPath string outputFormat string grpcTimeout time.Duration + appCtx context.Context + appCancel context.CancelFunc ) func init() { - cliCmd.PersistentFlags().StringVar(&inetMockSocketPath, "socket-path", "./inetmock.sock", "Path to the INetMock socket file") + cliCmd.PersistentFlags().StringVar(&inetMockSocketPath, "socket-path", "unix:///var/run/inetmock.sock", "Path to the INetMock socket file") cliCmd.PersistentFlags().StringVarP(&outputFormat, "format", "f", "table", "Output format to use. Possible values: table, json, yaml") cliCmd.PersistentFlags().DurationVar(&grpcTimeout, "grpc-timeout", 5*time.Second, "Timeout to connect to the gRPC API") - cliCmd.AddCommand(endpointsCmd, handlerCmd, healthCmd) + cliCmd.AddCommand(endpointsCmd, handlerCmd, healthCmd, auditCmd) endpointsCmd.AddCommand(getEndpoints) handlerCmd.AddCommand(getHandlersCmd) healthCmd.AddCommand(generalHealthCmd) healthCmd.AddCommand(containerHealthCmd) + + currentUser := "" + if usr, err := user.Current(); err == nil { + currentUser = usr.Username + } else { + currentUser = uuid.New().String() + } + + watchEventsCmd.PersistentFlags().StringVar( + &listenerName, + "listener-name", + currentUser, + "set listener name - defaults to the current username, if the user cannot be determined a random UUID will be used", + ) + auditCmd.AddCommand(watchEventsCmd, addFileCmd, removeFileCmd) + + appCtx, appCancel = initAppContext() } func ExecuteClientCommand() error { + defer appCancel() return cliCmd.Execute() } + +func initAppContext() (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + go func() { + <-signals + cancel() + }() + + return ctx, cancel +} diff --git a/internal/cmd/endpoints.go b/internal/cmd/endpoints.go index 3cd2749..f9dcd0c 100644 --- a/internal/cmd/endpoints.go +++ b/internal/cmd/endpoints.go @@ -15,7 +15,7 @@ var ( getEndpoints = &cobra.Command{ Use: "get", Short: "Get all running endpoints", - Run: runGetEndpoints, + RunE: runGetEndpoints, } endpointsCmd = &cobra.Command{ @@ -50,8 +50,7 @@ func fromEndpoints(eps []*rpc.Endpoint) (out []*printableEndpoint) { return } -func runGetEndpoints(_ *cobra.Command, _ []string) { - var err error +func runGetEndpoints(_ *cobra.Command, _ []string) (err error) { var conn *grpc.ClientConn if conn, err = grpc.Dial(inetMockSocketPath, grpc.WithInsecure()); err != nil { @@ -59,7 +58,8 @@ func runGetEndpoints(_ *cobra.Command, _ []string) { os.Exit(10) } endpointsClient := rpc.NewEndpointsClient(conn) - ctx, _ := context.WithTimeout(context.Background(), grpcTimeout) + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + defer cancel() var endpointsResp *rpc.GetEndpointsResponse if endpointsResp, err = endpointsClient.GetEndpoints(ctx, &rpc.GetEndpointsRequest{}); err != nil { fmt.Printf("Failed to get the endpoints: %v", err) @@ -70,4 +70,5 @@ func runGetEndpoints(_ *cobra.Command, _ []string) { if err = writer.Write(fromEndpoints(endpointsResp.Endpoints)); err != nil { fmt.Printf("Error occurred during writing response values: %v\n", err) } + return } diff --git a/internal/format/table_writer.go b/internal/format/table_writer.go index feb3383..da75f16 100644 --- a/internal/format/table_writer.go +++ b/internal/format/table_writer.go @@ -75,6 +75,15 @@ func (t *tblWriter) getData(val reflect.Value, numberOfFields int) (data []strin } func value(val reflect.Value) string { + + if val.IsZero() { + return "" + } + + if stringer, isStringer := val.Interface().(fmt.Stringer); isStringer { + return stringer.String() + } + switch val.Kind() { case reflect.Ptr: return value(val.Elem()) @@ -84,6 +93,8 @@ func value(val reflect.Value) string { return strconv.FormatBool(val.Bool()) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return strconv.FormatInt(val.Int(), 10) + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return strconv.FormatUint(val.Uint(), 10) case reflect.Float32, reflect.Float64: return strconv.FormatFloat(val.Float(), 'f', 6, 64) default: diff --git a/internal/rpc/audit_server.go b/internal/rpc/audit_server.go new file mode 100644 index 0000000..db8d1fa --- /dev/null +++ b/internal/rpc/audit_server.go @@ -0,0 +1,51 @@ +package rpc + +import ( + "context" + "io" + "os" + + "gitlab.com/inetmock/inetmock/internal/app" + "gitlab.com/inetmock/inetmock/pkg/audit" + "gitlab.com/inetmock/inetmock/pkg/audit/sink" + "go.uber.org/zap" +) + +type auditServer struct { + UnimplementedAuditServer + app app.App +} + +func (a *auditServer) WatchEvents(req *WatchEventsRequest, srv Audit_WatchEventsServer) (err error) { + a.app.Logger().Info("watcher attached", zap.String("name", req.WatcherName)) + err = a.app.EventStream().RegisterSink(sink.NewGRPCSink(srv.Context(), req.WatcherName, func(ev audit.Event) { + if err = srv.Send(ev.ProtoMessage()); err != nil { + return + } + })) + + if err != nil { + return + } + + <-srv.Context().Done() + a.app.Logger().Info("Watcher detached", zap.String("name", req.WatcherName)) + return +} + +func (a *auditServer) RegisterFileSink(_ context.Context, req *RegisterFileSinkRequest) (resp *RegisterFileSinkResponse, err error) { + var writer io.WriteCloser + if writer, err = os.OpenFile(req.TargetPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644); err != nil { + return + } + if err = a.app.EventStream().RegisterSink(sink.NewWriterSink(req.TargetPath, audit.NewEventWriter(writer))); err != nil { + return + } + resp = &RegisterFileSinkResponse{} + return +} + +func (a *auditServer) RemoveFileSink(_ context.Context, req *RemoveFileSinkRequest) (*RemoveFileSinkResponse, error) { + a.app.EventStream().RemoveSink(req.TargetPath) + return &RemoveFileSinkResponse{}, nil +} diff --git a/internal/rpc/grpc_api.go b/internal/rpc/grpc_api.go index cbcc87c..6c71d8c 100644 --- a/internal/rpc/grpc_api.go +++ b/internal/rpc/grpc_api.go @@ -52,6 +52,10 @@ func (i *inetmockAPI) StartServer() (err error) { app: i.app, }) + RegisterAuditServer(i.server, &auditServer{ + app: i.app, + }) + go i.startServerAsync(lis) return } diff --git a/internal/rpc/handlers_server.go b/internal/rpc/handlers_server.go index bd5491e..0af6e56 100644 --- a/internal/rpc/handlers_server.go +++ b/internal/rpc/handlers_server.go @@ -11,9 +11,6 @@ type handlersServer struct { registry api.HandlerRegistry } -func (h *handlersServer) mustEmbedUnimplementedHandlersServer() { -} - func (h *handlersServer) GetHandlers(_ context.Context, _ *GetHandlersRequest) (*GetHandlersResponse, error) { return &GetHandlersResponse{ Handlers: h.registry.AvailableHandlers(), diff --git a/internal/rpc/health_server.go b/internal/rpc/health_server.go index d92f344..17b0af7 100644 --- a/internal/rpc/health_server.go +++ b/internal/rpc/health_server.go @@ -3,12 +3,12 @@ package rpc import ( "context" - app2 "gitlab.com/inetmock/inetmock/internal/app" + "gitlab.com/inetmock/inetmock/internal/app" ) type healthServer struct { UnimplementedHealthServer - app app2.App + app app.App } func (h healthServer) GetHealth(_ context.Context, _ *HealthRequest) (resp *HealthResponse, err error) { diff --git a/pkg/audit/api.go b/pkg/audit/api.go index fa4ef7b..d269121 100644 --- a/pkg/audit/api.go +++ b/pkg/audit/api.go @@ -15,9 +15,11 @@ type Emitter interface { Emit(ev Event) } +type CloseHandle func() + type Sink interface { Name() string - OnSubscribe(evs <-chan Event) + OnSubscribe(evs <-chan Event, close CloseHandle) } type EventStream interface { @@ -25,4 +27,5 @@ type EventStream interface { Emitter RegisterSink(s Sink) error Sinks() []string + RemoveSink(name string) } diff --git a/pkg/audit/event.go b/pkg/audit/event.go index 0ec4aa5..5e2f29d 100644 --- a/pkg/audit/event.go +++ b/pkg/audit/event.go @@ -29,7 +29,7 @@ type Event struct { TLS *TLSDetails } -func (e *Event) ProtoMessage() proto.Message { +func (e *Event) ProtoMessage() *EventEntity { var sourceIP isEventEntity_SourceIP if ipv4 := e.SourceIP.To4(); ipv4 != nil { sourceIP = &EventEntity_SourceIPv4{SourceIPv4: ipv4ToUint32(ipv4)} diff --git a/pkg/audit/event_stream.go b/pkg/audit/event_stream.go index 51f6c7c..d3986b0 100644 --- a/pkg/audit/event_stream.go +++ b/pkg/audit/event_stream.go @@ -72,6 +72,20 @@ func (e *eventStream) Emit(ev Event) { } } +func (e *eventStream) RemoveSink(name string) { + e.lock.Lock() + defer e.lock.Unlock() + + sink, exists := e.sinks[name] + if !exists { + return + } + sink.lock.Lock() + defer sink.lock.Unlock() + delete(e.sinks, name) + close(sink.downstream) +} + func (e *eventStream) RegisterSink(s Sink) error { name := s.Name() @@ -83,7 +97,11 @@ func (e *eventStream) RegisterSink(s Sink) error { downstream: make(chan Event, e.sinkBufferSize), lock: new(sync.Mutex), } - s.OnSubscribe(rs.downstream) + + s.OnSubscribe(rs.downstream, func() { + e.RemoveSink(name) + }) + e.sinks[name] = rs return nil } diff --git a/pkg/audit/event_stream_test.go b/pkg/audit/event_stream_test.go index 1d47fdc..f89c981 100644 --- a/pkg/audit/event_stream_test.go +++ b/pkg/audit/event_stream_test.go @@ -61,7 +61,7 @@ func (t *testSink) Name() string { return t.name } -func (t *testSink) OnSubscribe(evs <-chan audit.Event) { +func (t *testSink) OnSubscribe(evs <-chan audit.Event, _ audit.CloseHandle) { go func() { for ev := range evs { if t.consumer != nil { diff --git a/pkg/audit/sink/grpc_sink.go b/pkg/audit/sink/grpc_sink.go new file mode 100644 index 0000000..de52156 --- /dev/null +++ b/pkg/audit/sink/grpc_sink.go @@ -0,0 +1,39 @@ +package sink + +import ( + "context" + + "gitlab.com/inetmock/inetmock/pkg/audit" +) + +func NewGRPCSink(ctx context.Context, name string, consumer func(ev audit.Event)) audit.Sink { + return &grpcSink{ + name: name, + ctx: ctx, + consumer: consumer, + } +} + +type grpcSink struct { + name string + ctx context.Context + consumer func(ev audit.Event) +} + +func (g grpcSink) Name() string { + return g.name +} + +func (g grpcSink) OnSubscribe(evs <-chan audit.Event, handle audit.CloseHandle) { + go func(ctx context.Context, consumer func(ev audit.Event), evs <-chan audit.Event, handle audit.CloseHandle) { + for { + select { + case ev := <-evs: + consumer(ev) + case <-ctx.Done(): + handle() + return + } + } + }(g.ctx, g.consumer, evs, handle) +} diff --git a/pkg/audit/sink/log_sink.go b/pkg/audit/sink/log_sink.go index 9b9de78..fb3c131 100644 --- a/pkg/audit/sink/log_sink.go +++ b/pkg/audit/sink/log_sink.go @@ -26,7 +26,7 @@ func (logSink) Name() string { return logSinkName } -func (l logSink) OnSubscribe(evs <-chan audit.Event) { +func (l logSink) OnSubscribe(evs <-chan audit.Event, _ audit.CloseHandle) { go func(logger logging.Logger, evs <-chan audit.Event) { for ev := range evs { eventLogger := logger diff --git a/pkg/audit/sink/writer_sink.go b/pkg/audit/sink/writer_sink.go index e0e2ba5..5409dde 100644 --- a/pkg/audit/sink/writer_sink.go +++ b/pkg/audit/sink/writer_sink.go @@ -33,7 +33,7 @@ func (f writerCloserSink) Name() string { return f.name } -func (f writerCloserSink) OnSubscribe(evs <-chan audit.Event) { +func (f writerCloserSink) OnSubscribe(evs <-chan audit.Event, _ audit.CloseHandle) { go func(target audit.Writer, closeOnExit bool, evs <-chan audit.Event) { for ev := range evs { _ = target.Write(&ev) From 2f0f3edfdf410c3eb50425265cbe83282842533a Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Sat, 16 Jan 2021 17:34:50 +0100 Subject: [PATCH 09/26] Moved code of commands to their main package - add init code to reduce code duplication of connection setup --- .goreleaser.yml | 6 ++-- {internal/cmd => cmd/imctl}/audit_file.go | 9 +----- {internal/cmd => cmd/imctl}/audit_watch.go | 10 +----- {internal/cmd => cmd/imctl}/cli.go | 29 +++++++++-------- {internal/cmd => cmd/imctl}/endpoints.go | 9 +----- {internal/cmd => cmd/imctl}/handlers.go | 15 +++------ {internal/cmd => cmd/imctl}/health.go | 12 ++----- cmd/imctl/main.go | 5 ++- {internal/cmd => cmd/inetmock}/ca.go | 2 +- cmd/inetmock/main.go | 37 ++++++++++++++++++++-- {internal/cmd => cmd/inetmock}/serve.go | 2 +- internal/cmd/server.go | 34 -------------------- pkg/audit/options.go | 8 ++++- 13 files changed, 75 insertions(+), 103 deletions(-) rename {internal/cmd => cmd/imctl}/audit_file.go (85%) rename {internal/cmd => cmd/imctl}/audit_watch.go (82%) rename {internal/cmd => cmd/imctl}/cli.go (74%) rename {internal/cmd => cmd/imctl}/endpoints.go (87%) rename {internal/cmd => cmd/imctl}/handlers.go (80%) rename {internal/cmd => cmd/imctl}/health.go (91%) rename {internal/cmd => cmd/inetmock}/ca.go (99%) rename {internal/cmd => cmd/inetmock}/serve.go (99%) delete mode 100644 internal/cmd/server.go diff --git a/.goreleaser.yml b/.goreleaser.yml index e426e71..027d4f3 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -7,7 +7,7 @@ before: builds: - id: "inetmock" binary: inetmock - main: ./cmd/inetmock/main.go + main: ./cmd/inetmock/ ldflags: - -w -s env: @@ -19,7 +19,7 @@ builds: - amd64 - id: "imctl" binary: imctl - main: ./cmd/imctl/main.go + main: ./cmd/imctl/ ldflags: - -w -s goos: @@ -65,7 +65,7 @@ release: name: inetmock dockers: - - binaries: + - ids: - inetmock - imctl image_templates: diff --git a/internal/cmd/audit_file.go b/cmd/imctl/audit_file.go similarity index 85% rename from internal/cmd/audit_file.go rename to cmd/imctl/audit_file.go index 9f98069..f34e4c2 100644 --- a/internal/cmd/audit_file.go +++ b/cmd/imctl/audit_file.go @@ -1,4 +1,4 @@ -package cmd +package main import ( "context" @@ -43,13 +43,6 @@ func runAddFile(_ *cobra.Command, args []string) (err error) { } func runRemoveFile(_ *cobra.Command, args []string) (err error) { - var conn *grpc.ClientConn - - if conn, err = grpc.Dial(inetMockSocketPath, grpc.WithInsecure()); err != nil { - fmt.Printf("Failed to connecto INetMock socket: %v\n", err) - os.Exit(10) - } - auditClient := rpc.NewAuditClient(conn) ctx, cancel := context.WithTimeout(appCtx, grpcTimeout) defer cancel() diff --git a/internal/cmd/audit_watch.go b/cmd/imctl/audit_watch.go similarity index 82% rename from internal/cmd/audit_watch.go rename to cmd/imctl/audit_watch.go index 93e775e..8bce566 100644 --- a/internal/cmd/audit_watch.go +++ b/cmd/imctl/audit_watch.go @@ -1,4 +1,4 @@ -package cmd +package main import ( "encoding/json" @@ -8,7 +8,6 @@ import ( "github.com/spf13/cobra" "gitlab.com/inetmock/inetmock/internal/rpc" "gitlab.com/inetmock/inetmock/pkg/audit" - "google.golang.org/grpc" ) var ( @@ -27,13 +26,6 @@ var ( ) func watchAuditEvents(_ *cobra.Command, _ []string) (err error) { - var conn *grpc.ClientConn - - if conn, err = grpc.Dial(inetMockSocketPath, grpc.WithInsecure()); err != nil { - fmt.Printf("Failed to connecto INetMock socket: %v\n", err) - os.Exit(10) - } - auditClient := rpc.NewAuditClient(conn) var watchClient rpc.Audit_WatchEventsClient diff --git a/internal/cmd/cli.go b/cmd/imctl/cli.go similarity index 74% rename from internal/cmd/cli.go rename to cmd/imctl/cli.go index bf89ede..a06057f 100644 --- a/internal/cmd/cli.go +++ b/cmd/imctl/cli.go @@ -1,4 +1,4 @@ -package cmd +package main import ( "context" @@ -10,12 +10,16 @@ import ( "github.com/google/uuid" "github.com/spf13/cobra" + "google.golang.org/grpc" ) var ( cliCmd = &cobra.Command{ Use: "", Short: "IMCTL is the CLI app to interact with an INetMock server", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return initGRPCConnection() + }, } inetMockSocketPath string @@ -23,6 +27,7 @@ var ( grpcTimeout time.Duration appCtx context.Context appCancel context.CancelFunc + conn *grpc.ClientConn ) func init() { @@ -33,8 +38,7 @@ func init() { cliCmd.AddCommand(endpointsCmd, handlerCmd, healthCmd, auditCmd) endpointsCmd.AddCommand(getEndpoints) handlerCmd.AddCommand(getHandlersCmd) - healthCmd.AddCommand(generalHealthCmd) - healthCmd.AddCommand(containerHealthCmd) + healthCmd.AddCommand(generalHealthCmd, containerHealthCmd) currentUser := "" if usr, err := user.Current(); err == nil { @@ -50,25 +54,22 @@ func init() { "set listener name - defaults to the current username, if the user cannot be determined a random UUID will be used", ) auditCmd.AddCommand(watchEventsCmd, addFileCmd, removeFileCmd) - - appCtx, appCancel = initAppContext() } -func ExecuteClientCommand() error { - defer appCancel() - return cliCmd.Execute() -} - -func initAppContext() (context.Context, context.CancelFunc) { - ctx, cancel := context.WithCancel(context.Background()) +func initGRPCConnection() (err error) { + appCtx, appCancel = context.WithCancel(context.Background()) signals := make(chan os.Signal, 1) signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) go func() { <-signals - cancel() + appCancel() }() - return ctx, cancel + dialCtx, cancel := context.WithTimeout(appCtx, grpcTimeout) + conn, err = grpc.DialContext(dialCtx, inetMockSocketPath, grpc.WithInsecure()) + cancel() + + return } diff --git a/internal/cmd/endpoints.go b/cmd/imctl/endpoints.go similarity index 87% rename from internal/cmd/endpoints.go rename to cmd/imctl/endpoints.go index f9dcd0c..d8c7d3e 100644 --- a/internal/cmd/endpoints.go +++ b/cmd/imctl/endpoints.go @@ -1,4 +1,4 @@ -package cmd +package main import ( "context" @@ -8,7 +8,6 @@ import ( "github.com/spf13/cobra" "gitlab.com/inetmock/inetmock/internal/format" "gitlab.com/inetmock/inetmock/internal/rpc" - "google.golang.org/grpc" ) var ( @@ -51,12 +50,6 @@ func fromEndpoints(eps []*rpc.Endpoint) (out []*printableEndpoint) { } func runGetEndpoints(_ *cobra.Command, _ []string) (err error) { - var conn *grpc.ClientConn - - if conn, err = grpc.Dial(inetMockSocketPath, grpc.WithInsecure()); err != nil { - fmt.Printf("Failed to connecto INetMock socket: %v\n", err) - os.Exit(10) - } endpointsClient := rpc.NewEndpointsClient(conn) ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) defer cancel() diff --git a/internal/cmd/handlers.go b/cmd/imctl/handlers.go similarity index 80% rename from internal/cmd/handlers.go rename to cmd/imctl/handlers.go index 136829c..7672e26 100644 --- a/internal/cmd/handlers.go +++ b/cmd/imctl/handlers.go @@ -1,4 +1,4 @@ -package cmd +package main import ( "context" @@ -8,7 +8,6 @@ import ( "github.com/spf13/cobra" "gitlab.com/inetmock/inetmock/internal/format" "gitlab.com/inetmock/inetmock/internal/rpc" - "google.golang.org/grpc" ) var ( @@ -39,15 +38,11 @@ func fromHandlers(hs []string) (handlers []*printableHandler) { } func runGetHandlers(_ *cobra.Command, _ []string) { - var err error - var conn *grpc.ClientConn - - if conn, err = grpc.Dial(inetMockSocketPath, grpc.WithInsecure()); err != nil { - fmt.Printf("Failed to connecto INetMock socket: %v\n", err) - os.Exit(10) - } handlersClient := rpc.NewHandlersClient(conn) - ctx, _ := context.WithTimeout(context.Background(), grpcTimeout) + + ctx, cancel := context.WithTimeout(appCtx, grpcTimeout) + defer cancel() + var err error var handlersResp *rpc.GetHandlersResponse if handlersResp, err = handlersClient.GetHandlers(ctx, &rpc.GetHandlersRequest{}); err != nil { diff --git a/internal/cmd/health.go b/cmd/imctl/health.go similarity index 91% rename from internal/cmd/health.go rename to cmd/imctl/health.go index 2b17d8f..e19b1c8 100644 --- a/internal/cmd/health.go +++ b/cmd/imctl/health.go @@ -1,4 +1,4 @@ -package cmd +package main import ( "context" @@ -8,7 +8,6 @@ import ( "github.com/spf13/cobra" "gitlab.com/inetmock/inetmock/internal/format" "gitlab.com/inetmock/inetmock/internal/rpc" - "google.golang.org/grpc" ) var ( @@ -55,15 +54,10 @@ func fromComponentsHealth(componentsHealth map[string]*rpc.ComponentHealth) (com } func getHealthResult() (healthResp *rpc.HealthResponse, err error) { - var conn *grpc.ClientConn - - if conn, err = grpc.Dial(inetMockSocketPath, grpc.WithInsecure()); err != nil { - return - } - var healthClient = rpc.NewHealthClient(conn) - ctx, _ := context.WithTimeout(context.Background(), grpcTimeout) + ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) healthResp, err = healthClient.GetHealth(ctx, &rpc.HealthRequest{}) + cancel() return } diff --git a/cmd/imctl/main.go b/cmd/imctl/main.go index 1771ede..822594e 100644 --- a/cmd/imctl/main.go +++ b/cmd/imctl/main.go @@ -1,9 +1,8 @@ package main -import "gitlab.com/inetmock/inetmock/internal/cmd" - func main() { - if err := cmd.ExecuteClientCommand(); err != nil { + defer appCancel() + if err := cliCmd.Execute(); err != nil { panic(err) } } diff --git a/internal/cmd/ca.go b/cmd/inetmock/ca.go similarity index 99% rename from internal/cmd/ca.go rename to cmd/inetmock/ca.go index df84505..1f54b47 100644 --- a/internal/cmd/ca.go +++ b/cmd/inetmock/ca.go @@ -1,4 +1,4 @@ -package cmd +package main import ( "crypto/tls" diff --git a/cmd/inetmock/main.go b/cmd/inetmock/main.go index 38cf492..50f9009 100644 --- a/cmd/inetmock/main.go +++ b/cmd/inetmock/main.go @@ -1,14 +1,47 @@ package main import ( - "gitlab.com/inetmock/inetmock/internal/cmd" + "fmt" + "os" + + "gitlab.com/inetmock/inetmock/internal/app" + "gitlab.com/inetmock/inetmock/internal/endpoint/handler/dns/mock" _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/dns/mock" _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/mock" + mock2 "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/mock" + "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/proxy" _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/proxy" + "gitlab.com/inetmock/inetmock/internal/endpoint/handler/metrics" _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/metrics" + "gitlab.com/inetmock/inetmock/internal/endpoint/handler/tls/interceptor" _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/tls/interceptor" + "go.uber.org/zap" +) + +var ( + server app.App ) func main() { - cmd.ExecuteServerCommand() + logger, _ := zap.NewProduction() + defer func() { + if err := logger.Sync(); err != nil { + fmt.Println(err.Error()) + } + }() + + var err error + if server, err = app.NewApp( + mock2.AddHTTPMock, + mock.AddDNSMock, + interceptor.AddTLSInterceptor, + proxy.AddHTTPProxy, + metrics.AddMetricsExporter, + ); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + server. + WithCommands(serveCmd, generateCaCmd). + MustRun() } diff --git a/internal/cmd/serve.go b/cmd/inetmock/serve.go similarity index 99% rename from internal/cmd/serve.go rename to cmd/inetmock/serve.go index 97ce54f..cff2dff 100644 --- a/internal/cmd/serve.go +++ b/cmd/inetmock/serve.go @@ -1,4 +1,4 @@ -package cmd +package main import ( "strings" diff --git a/internal/cmd/server.go b/internal/cmd/server.go deleted file mode 100644 index d3d9bc1..0000000 --- a/internal/cmd/server.go +++ /dev/null @@ -1,34 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - - "gitlab.com/inetmock/inetmock/internal/app" - "gitlab.com/inetmock/inetmock/internal/endpoint/handler/dns/mock" - mock2 "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/mock" - "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/proxy" - "gitlab.com/inetmock/inetmock/internal/endpoint/handler/metrics" - "gitlab.com/inetmock/inetmock/internal/endpoint/handler/tls/interceptor" -) - -var ( - server app.App -) - -func ExecuteServerCommand() { - var err error - if server, err = app.NewApp( - mock2.AddHTTPMock, - mock.AddDNSMock, - interceptor.AddTLSInterceptor, - proxy.AddHTTPProxy, - metrics.AddMetricsExporter, - ); err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - server. - WithCommands(serveCmd, generateCaCmd). - MustRun() -} diff --git a/pkg/audit/options.go b/pkg/audit/options.go index 76b737b..46868e9 100644 --- a/pkg/audit/options.go +++ b/pkg/audit/options.go @@ -12,8 +12,8 @@ const ( ) var ( + defaultDistributeParallelization int generatorIdx int64 = 1 - defaultDistributeParallelization = runtime.NumCPU() / 2 WithBufferSize = func(bufferSize int) EventStreamOption { return func(cfg *eventStreamCfg) { cfg.bufferSize = bufferSize @@ -44,6 +44,12 @@ var ( } ) +func init() { + if defaultDistributeParallelization = runtime.NumCPU() / 2; defaultDistributeParallelization < 2 { + defaultDistributeParallelization = 2 + } +} + type EventStreamOption func(cfg *eventStreamCfg) type eventStreamCfg struct { From 41aa132e80178121038e7fafc7f0bf05bc275f3a Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Sat, 16 Jan 2021 17:43:54 +0100 Subject: [PATCH 10/26] Cache .task/ directory to lazy run tasks across stages --- .gitlab-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ec52ec8..efa1b49 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,11 @@ image: registry.gitlab.com/inetmock/ci-image +cache: &global_cache + key: ${CI_COMMIT_REF_SLUG} + paths: + - .task/ + policy: pull-push + stages: - test - build @@ -7,6 +13,8 @@ stages: test: stage: test + cache: + <<: *global_cache script: - task cli-cover-report artifacts: @@ -22,6 +30,8 @@ lint: snapshot-release: stage: build + cache: + <<: *global_cache services: - docker:dind before_script: @@ -33,6 +43,8 @@ snapshot-release: release: stage: release + cache: + <<: *global_cache services: - docker:dind only: From af5c55d8c9d84bab571844dd849c6986a37a343c Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Sat, 16 Jan 2021 17:49:20 +0100 Subject: [PATCH 11/26] Cache .task/ directory to lazy run tasks across stages --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index efa1b49..6baf78d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ cache: &global_cache key: ${CI_COMMIT_REF_SLUG} paths: - .task/ + - ./**/*.pb.go policy: pull-push stages: From 38293bb8c4f9f8f9c5a795995c6ee642eb71effb Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Sat, 16 Jan 2021 17:51:30 +0100 Subject: [PATCH 12/26] Remove caching for now --- .gitlab-ci.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6baf78d..ec52ec8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,12 +1,5 @@ image: registry.gitlab.com/inetmock/ci-image -cache: &global_cache - key: ${CI_COMMIT_REF_SLUG} - paths: - - .task/ - - ./**/*.pb.go - policy: pull-push - stages: - test - build @@ -14,8 +7,6 @@ stages: test: stage: test - cache: - <<: *global_cache script: - task cli-cover-report artifacts: @@ -31,8 +22,6 @@ lint: snapshot-release: stage: build - cache: - <<: *global_cache services: - docker:dind before_script: @@ -44,8 +33,6 @@ snapshot-release: release: stage: release - cache: - <<: *global_cache services: - docker:dind only: From 37c87adaf34cb1ef7ccc864b18624985c0e1d565 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Sat, 16 Jan 2021 18:26:15 +0100 Subject: [PATCH 13/26] Apply comments --- api/proto/internal/rpc/audit.proto | 2 +- cmd/imctl/endpoints.go | 2 +- internal/rpc/audit_server.go | 19 +++++++++++-------- internal/rpc/grpc_api.go | 3 ++- pkg/audit/api.go | 2 +- pkg/audit/event_stream.go | 7 +++++-- 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/api/proto/internal/rpc/audit.proto b/api/proto/internal/rpc/audit.proto index aa07869..2eb4a15 100644 --- a/api/proto/internal/rpc/audit.proto +++ b/api/proto/internal/rpc/audit.proto @@ -26,7 +26,7 @@ message RemoveFileSinkRequest { } message RemoveFileSinkResponse { - + bool SinkGotRemoved = 1; } service Audit { diff --git a/cmd/imctl/endpoints.go b/cmd/imctl/endpoints.go index d8c7d3e..997164d 100644 --- a/cmd/imctl/endpoints.go +++ b/cmd/imctl/endpoints.go @@ -51,7 +51,7 @@ func fromEndpoints(eps []*rpc.Endpoint) (out []*printableEndpoint) { func runGetEndpoints(_ *cobra.Command, _ []string) (err error) { endpointsClient := rpc.NewEndpointsClient(conn) - ctx, cancel := context.WithTimeout(context.Background(), grpcTimeout) + ctx, cancel := context.WithTimeout(appCtx, grpcTimeout) defer cancel() var endpointsResp *rpc.GetEndpointsResponse if endpointsResp, err = endpointsClient.GetEndpoints(ctx, &rpc.GetEndpointsRequest{}); err != nil { diff --git a/internal/rpc/audit_server.go b/internal/rpc/audit_server.go index db8d1fa..7cb59d2 100644 --- a/internal/rpc/audit_server.go +++ b/internal/rpc/audit_server.go @@ -5,20 +5,21 @@ import ( "io" "os" - "gitlab.com/inetmock/inetmock/internal/app" "gitlab.com/inetmock/inetmock/pkg/audit" "gitlab.com/inetmock/inetmock/pkg/audit/sink" + "gitlab.com/inetmock/inetmock/pkg/logging" "go.uber.org/zap" ) type auditServer struct { UnimplementedAuditServer - app app.App + logger logging.Logger + eventStream audit.EventStream } func (a *auditServer) WatchEvents(req *WatchEventsRequest, srv Audit_WatchEventsServer) (err error) { - a.app.Logger().Info("watcher attached", zap.String("name", req.WatcherName)) - err = a.app.EventStream().RegisterSink(sink.NewGRPCSink(srv.Context(), req.WatcherName, func(ev audit.Event) { + a.logger.Info("watcher attached", zap.String("name", req.WatcherName)) + err = a.eventStream.RegisterSink(sink.NewGRPCSink(srv.Context(), req.WatcherName, func(ev audit.Event) { if err = srv.Send(ev.ProtoMessage()); err != nil { return } @@ -29,7 +30,7 @@ func (a *auditServer) WatchEvents(req *WatchEventsRequest, srv Audit_WatchEvents } <-srv.Context().Done() - a.app.Logger().Info("Watcher detached", zap.String("name", req.WatcherName)) + a.logger.Info("Watcher detached", zap.String("name", req.WatcherName)) return } @@ -38,7 +39,7 @@ func (a *auditServer) RegisterFileSink(_ context.Context, req *RegisterFileSinkR if writer, err = os.OpenFile(req.TargetPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644); err != nil { return } - if err = a.app.EventStream().RegisterSink(sink.NewWriterSink(req.TargetPath, audit.NewEventWriter(writer))); err != nil { + if err = a.eventStream.RegisterSink(sink.NewWriterSink(req.TargetPath, audit.NewEventWriter(writer))); err != nil { return } resp = &RegisterFileSinkResponse{} @@ -46,6 +47,8 @@ func (a *auditServer) RegisterFileSink(_ context.Context, req *RegisterFileSinkR } func (a *auditServer) RemoveFileSink(_ context.Context, req *RemoveFileSinkRequest) (*RemoveFileSinkResponse, error) { - a.app.EventStream().RemoveSink(req.TargetPath) - return &RemoveFileSinkResponse{}, nil + gotRemoved := a.eventStream.RemoveSink(req.TargetPath) + return &RemoveFileSinkResponse{ + SinkGotRemoved: gotRemoved, + }, nil } diff --git a/internal/rpc/grpc_api.go b/internal/rpc/grpc_api.go index 6c71d8c..b5f0f65 100644 --- a/internal/rpc/grpc_api.go +++ b/internal/rpc/grpc_api.go @@ -53,7 +53,8 @@ func (i *inetmockAPI) StartServer() (err error) { }) RegisterAuditServer(i.server, &auditServer{ - app: i.app, + logger: i.app.Logger(), + eventStream: i.app.EventStream(), }) go i.startServerAsync(lis) diff --git a/pkg/audit/api.go b/pkg/audit/api.go index d269121..4129a2d 100644 --- a/pkg/audit/api.go +++ b/pkg/audit/api.go @@ -27,5 +27,5 @@ type EventStream interface { Emitter RegisterSink(s Sink) error Sinks() []string - RemoveSink(name string) + RemoveSink(name string) (exists bool) } diff --git a/pkg/audit/event_stream.go b/pkg/audit/event_stream.go index d3986b0..6dbd522 100644 --- a/pkg/audit/event_stream.go +++ b/pkg/audit/event_stream.go @@ -72,11 +72,12 @@ func (e *eventStream) Emit(ev Event) { } } -func (e *eventStream) RemoveSink(name string) { +func (e *eventStream) RemoveSink(name string) (exists bool) { e.lock.Lock() defer e.lock.Unlock() - sink, exists := e.sinks[name] + var sink *registeredSink + sink, exists = e.sinks[name] if !exists { return } @@ -84,6 +85,8 @@ func (e *eventStream) RemoveSink(name string) { defer sink.lock.Unlock() delete(e.sinks, name) close(sink.downstream) + + return } func (e *eventStream) RegisterSink(s Sink) error { From 0138b5778251546328845c9095d528e4d58bde28 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Mon, 18 Jan 2021 18:35:13 +0100 Subject: [PATCH 14/26] Merged CLI and server app init --- api/proto/internal/rpc/audit.proto | 7 + cmd/imctl/audit_file.go | 15 +- cmd/imctl/audit_watch.go | 4 +- cmd/imctl/cli.go | 75 --------- cmd/imctl/endpoints.go | 2 +- cmd/imctl/handlers.go | 2 +- cmd/imctl/main.go | 61 ++++++- cmd/inetmock/main.go | 35 ++-- go.mod | 1 + internal/app/app.go | 249 ++++++++++++++++++++++------- internal/rpc/audit_server.go | 16 +- 11 files changed, 302 insertions(+), 165 deletions(-) delete mode 100644 cmd/imctl/cli.go diff --git a/api/proto/internal/rpc/audit.proto b/api/proto/internal/rpc/audit.proto index 2eb4a15..7494a5c 100644 --- a/api/proto/internal/rpc/audit.proto +++ b/api/proto/internal/rpc/audit.proto @@ -9,12 +9,19 @@ import 'pkg/audit/event_entity.proto'; package inetmock.rpc; +enum FileOpenMode { + TRUNCATE = 0; + APPEND = 1; +} + message WatchEventsRequest { string watcherName = 1; } message RegisterFileSinkRequest { string targetPath = 1; + FileOpenMode openMode = 2; + uint32 permissions = 3; } message RegisterFileSinkResponse { diff --git a/cmd/imctl/audit_file.go b/cmd/imctl/audit_file.go index f34e4c2..132067f 100644 --- a/cmd/imctl/audit_file.go +++ b/cmd/imctl/audit_file.go @@ -24,6 +24,13 @@ var ( Args: cobra.ExactArgs(1), RunE: runRemoveFile, } + + readFileCmd = &cobra.Command{ + Use: "readFile", + Short: "reads an audit file and prints the events", + Args: cobra.ExactArgs(1), + RunE: runReadFile, + } ) func runAddFile(_ *cobra.Command, args []string) (err error) { @@ -35,7 +42,7 @@ func runAddFile(_ *cobra.Command, args []string) (err error) { } auditClient := rpc.NewAuditClient(conn) - ctx, cancel := context.WithTimeout(appCtx, grpcTimeout) + ctx, cancel := context.WithTimeout(cliApp.Context(), grpcTimeout) defer cancel() _, err = auditClient.RegisterFileSink(ctx, &rpc.RegisterFileSinkRequest{TargetPath: args[0]}) @@ -44,9 +51,13 @@ func runAddFile(_ *cobra.Command, args []string) (err error) { func runRemoveFile(_ *cobra.Command, args []string) (err error) { auditClient := rpc.NewAuditClient(conn) - ctx, cancel := context.WithTimeout(appCtx, grpcTimeout) + ctx, cancel := context.WithTimeout(cliApp.Context(), grpcTimeout) defer cancel() _, err = auditClient.RemoveFileSink(ctx, &rpc.RemoveFileSinkRequest{TargetPath: args[0]}) return } + +func runReadFile(_ *cobra.Command, args []string) (err error) { + return +} diff --git a/cmd/imctl/audit_watch.go b/cmd/imctl/audit_watch.go index 8bce566..42f4345 100644 --- a/cmd/imctl/audit_watch.go +++ b/cmd/imctl/audit_watch.go @@ -29,7 +29,7 @@ func watchAuditEvents(_ *cobra.Command, _ []string) (err error) { auditClient := rpc.NewAuditClient(conn) var watchClient rpc.Audit_WatchEventsClient - if watchClient, err = auditClient.WatchEvents(appCtx, &rpc.WatchEventsRequest{WatcherName: listenerName}); err != nil { + if watchClient, err = auditClient.WatchEvents(cliApp.Context(), &rpc.WatchEventsRequest{WatcherName: listenerName}); err != nil { fmt.Println(err.Error()) os.Exit(1) } @@ -47,7 +47,7 @@ func watchAuditEvents(_ *cobra.Command, _ []string) (err error) { } }() - <-appCtx.Done() + <-cliApp.Context().Done() err = watchClient.CloseSend() return diff --git a/cmd/imctl/cli.go b/cmd/imctl/cli.go deleted file mode 100644 index a06057f..0000000 --- a/cmd/imctl/cli.go +++ /dev/null @@ -1,75 +0,0 @@ -package main - -import ( - "context" - "os" - "os/signal" - "os/user" - "syscall" - "time" - - "github.com/google/uuid" - "github.com/spf13/cobra" - "google.golang.org/grpc" -) - -var ( - cliCmd = &cobra.Command{ - Use: "", - Short: "IMCTL is the CLI app to interact with an INetMock server", - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return initGRPCConnection() - }, - } - - inetMockSocketPath string - outputFormat string - grpcTimeout time.Duration - appCtx context.Context - appCancel context.CancelFunc - conn *grpc.ClientConn -) - -func init() { - cliCmd.PersistentFlags().StringVar(&inetMockSocketPath, "socket-path", "unix:///var/run/inetmock.sock", "Path to the INetMock socket file") - cliCmd.PersistentFlags().StringVarP(&outputFormat, "format", "f", "table", "Output format to use. Possible values: table, json, yaml") - cliCmd.PersistentFlags().DurationVar(&grpcTimeout, "grpc-timeout", 5*time.Second, "Timeout to connect to the gRPC API") - - cliCmd.AddCommand(endpointsCmd, handlerCmd, healthCmd, auditCmd) - endpointsCmd.AddCommand(getEndpoints) - handlerCmd.AddCommand(getHandlersCmd) - healthCmd.AddCommand(generalHealthCmd, containerHealthCmd) - - currentUser := "" - if usr, err := user.Current(); err == nil { - currentUser = usr.Username - } else { - currentUser = uuid.New().String() - } - - watchEventsCmd.PersistentFlags().StringVar( - &listenerName, - "listener-name", - currentUser, - "set listener name - defaults to the current username, if the user cannot be determined a random UUID will be used", - ) - auditCmd.AddCommand(watchEventsCmd, addFileCmd, removeFileCmd) -} - -func initGRPCConnection() (err error) { - appCtx, appCancel = context.WithCancel(context.Background()) - - signals := make(chan os.Signal, 1) - signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - - go func() { - <-signals - appCancel() - }() - - dialCtx, cancel := context.WithTimeout(appCtx, grpcTimeout) - conn, err = grpc.DialContext(dialCtx, inetMockSocketPath, grpc.WithInsecure()) - cancel() - - return -} diff --git a/cmd/imctl/endpoints.go b/cmd/imctl/endpoints.go index 997164d..592ef9b 100644 --- a/cmd/imctl/endpoints.go +++ b/cmd/imctl/endpoints.go @@ -51,7 +51,7 @@ func fromEndpoints(eps []*rpc.Endpoint) (out []*printableEndpoint) { func runGetEndpoints(_ *cobra.Command, _ []string) (err error) { endpointsClient := rpc.NewEndpointsClient(conn) - ctx, cancel := context.WithTimeout(appCtx, grpcTimeout) + ctx, cancel := context.WithTimeout(cliApp.Context(), grpcTimeout) defer cancel() var endpointsResp *rpc.GetEndpointsResponse if endpointsResp, err = endpointsClient.GetEndpoints(ctx, &rpc.GetEndpointsRequest{}); err != nil { diff --git a/cmd/imctl/handlers.go b/cmd/imctl/handlers.go index 7672e26..06822da 100644 --- a/cmd/imctl/handlers.go +++ b/cmd/imctl/handlers.go @@ -40,7 +40,7 @@ func fromHandlers(hs []string) (handlers []*printableHandler) { func runGetHandlers(_ *cobra.Command, _ []string) { handlersClient := rpc.NewHandlersClient(conn) - ctx, cancel := context.WithTimeout(appCtx, grpcTimeout) + ctx, cancel := context.WithTimeout(cliApp.Context(), grpcTimeout) defer cancel() var err error var handlersResp *rpc.GetHandlersResponse diff --git a/cmd/imctl/main.go b/cmd/imctl/main.go index 822594e..1ca965f 100644 --- a/cmd/imctl/main.go +++ b/cmd/imctl/main.go @@ -1,8 +1,63 @@ package main +import ( + "context" + "os/user" + "time" + + "github.com/google/uuid" + "github.com/spf13/cobra" + "gitlab.com/inetmock/inetmock/internal/app" + "google.golang.org/grpc" +) + +var ( + inetMockSocketPath string + outputFormat string + grpcTimeout time.Duration + cliApp app.App + conn *grpc.ClientConn +) + func main() { - defer appCancel() - if err := cliCmd.Execute(); err != nil { - panic(err) + + endpointsCmd.AddCommand(getEndpoints) + handlerCmd.AddCommand(getHandlersCmd) + healthCmd.AddCommand(generalHealthCmd, containerHealthCmd) + + cliApp = app.NewApp("imctl", "IMCTL is the CLI app to interact with an INetMock server"). + WithCommands(endpointsCmd, handlerCmd, healthCmd, auditCmd). + WithInitTasks(func(_ *cobra.Command, _ []string) (err error) { + return initGRPCConnection() + }) + + cliApp.RootCommand().PersistentFlags().StringVar(&inetMockSocketPath, "socket-path", "unix:///var/run/inetmock.sock", "Path to the INetMock socket file") + cliApp.RootCommand().PersistentFlags().StringVarP(&outputFormat, "format", "f", "table", "Output format to use. Possible values: table, json, yaml") + cliApp.RootCommand().PersistentFlags().DurationVar(&grpcTimeout, "grpc-timeout", 5*time.Second, "Timeout to connect to the gRPC API") + + currentUser := "" + if usr, err := user.Current(); err == nil { + currentUser = usr.Username + } else { + currentUser = uuid.New().String() } + + watchEventsCmd.PersistentFlags().StringVar( + &listenerName, + "listener-name", + currentUser, + "set listener name - defaults to the current username, if the user cannot be determined a random UUID will be used", + ) + auditCmd.AddCommand(watchEventsCmd, addFileCmd, removeFileCmd) + + cliApp.MustRun() + +} + +func initGRPCConnection() (err error) { + dialCtx, cancel := context.WithTimeout(cliApp.Context(), grpcTimeout) + conn, err = grpc.DialContext(dialCtx, inetMockSocketPath, grpc.WithInsecure()) + cancel() + + return } diff --git a/cmd/inetmock/main.go b/cmd/inetmock/main.go index 50f9009..bcec3ca 100644 --- a/cmd/inetmock/main.go +++ b/cmd/inetmock/main.go @@ -2,19 +2,13 @@ package main import ( "fmt" - "os" "gitlab.com/inetmock/inetmock/internal/app" - "gitlab.com/inetmock/inetmock/internal/endpoint/handler/dns/mock" - _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/dns/mock" - _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/mock" - mock2 "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/mock" + dns "gitlab.com/inetmock/inetmock/internal/endpoint/handler/dns/mock" + http "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/mock" "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/proxy" - _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/proxy" "gitlab.com/inetmock/inetmock/internal/endpoint/handler/metrics" - _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/metrics" "gitlab.com/inetmock/inetmock/internal/endpoint/handler/tls/interceptor" - _ "gitlab.com/inetmock/inetmock/internal/endpoint/handler/tls/interceptor" "go.uber.org/zap" ) @@ -30,18 +24,19 @@ func main() { } }() - var err error - if server, err = app.NewApp( - mock2.AddHTTPMock, - mock.AddDNSMock, - interceptor.AddTLSInterceptor, - proxy.AddHTTPProxy, - metrics.AddMetricsExporter, - ); err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - server. + app.NewApp("inetmock", "INetMock is lightweight internet mock"). + WithHandlerRegistry( + http.AddHTTPMock, + dns.AddDNSMock, + interceptor.AddTLSInterceptor, + proxy.AddHTTPProxy, + metrics.AddMetricsExporter). WithCommands(serveCmd, generateCaCmd). + WithConfig(). + WithLogger(). + WithHealthChecker(). + WithCertStore(). + WithEventStream(). + WithEndpointManager(). MustRun() } diff --git a/go.mod b/go.mod index 993562b..aff09f2 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 + go.uber.org/multierr v1.5.0 go.uber.org/zap v1.16.0 google.golang.org/grpc v1.34.0 google.golang.org/protobuf v1.25.0 diff --git a/internal/app/app.go b/internal/app/app.go index 3540e6c..1a8b6db 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -18,6 +18,7 @@ import ( "gitlab.com/inetmock/inetmock/pkg/health" "gitlab.com/inetmock/inetmock/pkg/logging" "gitlab.com/inetmock/inetmock/pkg/path" + "go.uber.org/multierr" "go.uber.org/zap" ) @@ -27,6 +28,18 @@ var ( developmentLogs bool ) +type contextKey string + +const ( + loggerKey contextKey = "gitlab.com/inetmock/inetmock/app/context/logger" + configKey contextKey = "gitlab.com/inetmock/inetmock/app/context/config" + handlerRegistryKey contextKey = "gitlab.com/inetmock/inetmock/app/context/handlerRegistry" + healthCheckerKey contextKey = "gitlab.com/inetmock/inetmock/app/context/healthChecker" + endpointManagerKey contextKey = "gitlab.com/inetmock/inetmock/app/context/endpointManager" + certStoreKey contextKey = "gitlab.com/inetmock/inetmock/app/context/certStore" + eventStreamKey contextKey = "gitlab.com/inetmock/inetmock/app/context/eventStream" +) + type App interface { api.PluginContext EventStream() audit.EventStream @@ -35,28 +48,56 @@ type App interface { EndpointManager() endpoint.EndpointManager HandlerRegistry() api.HandlerRegistry Context() context.Context + RootCommand() *cobra.Command MustRun() Shutdown() + + // WithCommands adds subcommands to the root command + // requires nothing WithCommands(cmds ...*cobra.Command) App + + // WithHandlerRegistry builds up the handler registry + // requires nothing + WithHandlerRegistry(registrations ...api.Registration) App + + // WithHealthChecker adds the health checker mechanism + // requires nothing + WithHealthChecker() App + + // WithLogger configures the logging system + // requires nothing + WithLogger() App + + // WithEndpointManager creates an endpoint manager instance and adds it to the context + // requires WithHandlerRegistry, WithHealthChecker and WithLogger + WithEndpointManager() App + + // WithCertStore initializes the cert store + // requires WithLogger and WithConfig + WithCertStore() App + + // WithEventStream adds the audit event stream + // requires WithLogger + WithEventStream() App + + // WithConfig loads the config + // requires nothing + WithConfig() App + + WithInitTasks(task ...func(cmd *cobra.Command, args []string) (err error)) App } type app struct { - cfg config.Config - rootCmd *cobra.Command - rootLogger logging.Logger - certStore cert.Store - checker health.Checker - endpointManager endpoint.EndpointManager - registry api.HandlerRegistry - ctx context.Context - cancel context.CancelFunc - eventStream audit.EventStream + rootCmd *cobra.Command + ctx context.Context + cancel context.CancelFunc + lateInitTasks []func(cmd *cobra.Command, args []string) (err error) } func (a *app) MustRun() { if err := a.rootCmd.Execute(); err != nil { - if a.rootLogger != nil { - a.rootLogger.Error( + if a.Logger() != nil { + a.Logger().Error( "Failed to run inetmock", zap.Error(err), ) @@ -66,107 +107,146 @@ func (a *app) MustRun() { } } -func (a app) Logger() logging.Logger { - return a.rootLogger +func (a *app) Logger() logging.Logger { + return a.ctx.Value(loggerKey).(logging.Logger) } -func (a app) Config() config.Config { - return a.cfg +func (a *app) Config() config.Config { + return a.ctx.Value(configKey).(config.Config) } -func (a app) CertStore() cert.Store { - return a.certStore +func (a *app) CertStore() cert.Store { + return a.ctx.Value(certStoreKey).(cert.Store) } -func (a app) Checker() health.Checker { - return a.checker +func (a *app) Checker() health.Checker { + return a.ctx.Value(healthCheckerKey).(health.Checker) } -func (a app) EndpointManager() endpoint.EndpointManager { - return a.endpointManager +func (a *app) EndpointManager() endpoint.EndpointManager { + return a.ctx.Value(endpointManagerKey).(endpoint.EndpointManager) } -func (a app) Audit() audit.Emitter { - return a.eventStream +func (a *app) Audit() audit.Emitter { + return a.ctx.Value(eventStreamKey).(audit.Emitter) } -func (a app) EventStream() audit.EventStream { - return a.eventStream +func (a *app) EventStream() audit.EventStream { + return a.ctx.Value(eventStreamKey).(audit.EventStream) } -func (a app) HandlerRegistry() api.HandlerRegistry { - return a.registry +func (a *app) HandlerRegistry() api.HandlerRegistry { + return a.ctx.Value(handlerRegistryKey).(api.HandlerRegistry) } -func (a app) Context() context.Context { +func (a *app) Context() context.Context { return a.ctx } -func (a app) Shutdown() { +func (a *app) RootCommand() *cobra.Command { + return a.rootCmd +} + +func (a *app) Shutdown() { a.cancel() } +// WithCommands adds subcommands to the root command +// requires nothing func (a *app) WithCommands(cmds ...*cobra.Command) App { a.rootCmd.AddCommand(cmds...) return a } -func NewApp(registrations ...api.Registration) (inetmockApp App, err error) { +// WithHandlerRegistry builds up the handler registry +// requires nothing +func (a *app) WithHandlerRegistry(registrations ...api.Registration) App { registry := api.NewHandlerRegistry() for _, registration := range registrations { - if err = registration(registry); err != nil { - return + if err := registration(registry); err != nil { + panic(err) } } - ctx, cancel := initAppContext() + a.ctx = context.WithValue(a.ctx, handlerRegistryKey, registry) - a := &app{ - rootCmd: &cobra.Command{ - Short: "INetMock is lightweight internet mock", - }, - checker: health.New(), - registry: registry, - ctx: ctx, - cancel: cancel, - } + return a +} - a.rootCmd.PersistentFlags().StringVar(&configFilePath, "config", "", "Path to config file that should be used") - a.rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", "logging level to use") - a.rootCmd.PersistentFlags().BoolVar(&developmentLogs, "development-logs", false, "Enable development mode logs") +// WithHealthChecker adds the health checker mechanism +// requires nothing +func (a *app) WithHealthChecker() App { + checker := health.New() + a.ctx = context.WithValue(a.ctx, healthCheckerKey, checker) + return a +} - a.rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) (err error) { +// WithLogger configures the logging system +// requires nothing +func (a *app) WithLogger() App { + a.lateInitTasks = append(a.lateInitTasks, func(cmd *cobra.Command, args []string) (err error) { logging.ConfigureLogging( logging.ParseLevel(logLevel), developmentLogs, map[string]interface{}{ - "cwd": path.WorkingDirectory(), + "cwd": path.WorkingDirectory(), + "cmd": cmd.Name(), + "args": args, }, ) - if a.rootLogger, err = logging.CreateLogger(); err != nil { + var logger logging.Logger + if logger, err = logging.CreateLogger(); err != nil { return } + a.ctx = context.WithValue(a.ctx, loggerKey, logger) + return + }) + return a +} - a.endpointManager = endpoint.NewEndpointManager( - a.registry, +// WithEndpointManager creates an endpoint manager instance and adds it to the context +// requires WithHandlerRegistry, WithHealthChecker and WithLogger +func (a *app) WithEndpointManager() App { + a.lateInitTasks = append(a.lateInitTasks, func(_ *cobra.Command, _ []string) (err error) { + epMgr := endpoint.NewEndpointManager( + a.HandlerRegistry(), a.Logger().Named("EndpointManager"), - a.checker, + a.Checker(), a, ) - a.cfg = config.CreateConfig(cmd.Flags()) + a.ctx = context.WithValue(a.ctx, endpointManagerKey, epMgr) + return + }) + return a +} - if err = a.cfg.ReadConfig(configFilePath); err != nil { +// WithCertStore initializes the cert store +// requires WithLogger and WithConfig +func (a *app) WithCertStore() App { + a.lateInitTasks = append(a.lateInitTasks, func(cmd *cobra.Command, args []string) (err error) { + var certStore cert.Store + if certStore, err = cert.NewDefaultStore( + a.Config(), + a.Logger().Named("CertStore"), + ); err != nil { return } - if a.certStore, err = cert.NewDefaultStore(a.cfg, a.rootLogger); err != nil { - return - } + a.ctx = context.WithValue(a.ctx, certStoreKey, certStore) + return + }) + return a +} - a.eventStream, err = audit.NewEventStream( +// WithEventStream adds the audit event stream +// requires WithLogger +func (a *app) WithEventStream() App { + a.lateInitTasks = append(a.lateInitTasks, func(_ *cobra.Command, _ []string) (err error) { + var eventStream audit.EventStream + eventStream, err = audit.NewEventStream( a.Logger().Named("EventStream"), audit.WithSinkBufferSize(10), ) @@ -174,11 +254,60 @@ func NewApp(registrations ...api.Registration) (inetmockApp App, err error) { return } - err = a.eventStream.RegisterSink(sink.NewLogSink(a.Logger().Named("LogSink"))) + if err = eventStream.RegisterSink(sink.NewLogSink(a.Logger().Named("LogSink"))); err != nil { + return + } + + a.ctx = context.WithValue(a.ctx, eventStreamKey, eventStream) + return + }) + + return a +} + +// WithConfig loads the config +// requires nothing +func (a *app) WithConfig() App { + a.lateInitTasks = append(a.lateInitTasks, func(cmd *cobra.Command, _ []string) (err error) { + cfg := config.CreateConfig(cmd.Flags()) + if err = cfg.ReadConfig(configFilePath); err != nil { + return + } + a.ctx = context.WithValue(a.ctx, configKey, cfg) + return + }) + + return a +} + +func (a *app) WithInitTasks(task ...func(cmd *cobra.Command, args []string) (err error)) App { + a.lateInitTasks = append(a.lateInitTasks, task...) + return a +} + +func NewApp(name, short string) App { + ctx, cancel := initAppContext() + a := &app{ + rootCmd: &cobra.Command{ + Use: name, + Short: short, + }, + ctx: ctx, + cancel: cancel, + } + + a.rootCmd.PersistentFlags().StringVar(&configFilePath, "config", "", "Path to config file that should be used") + a.rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", "logging level to use") + a.rootCmd.PersistentFlags().BoolVar(&developmentLogs, "development-logs", false, "Enable development mode logs") + + a.rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) (err error) { + for _, initTask := range a.lateInitTasks { + err = multierr.Append(err, initTask(cmd, args)) + } return } - return a, nil + return a } func initAppContext() (context.Context, context.CancelFunc) { diff --git a/internal/rpc/audit_server.go b/internal/rpc/audit_server.go index 7cb59d2..96bdc63 100644 --- a/internal/rpc/audit_server.go +++ b/internal/rpc/audit_server.go @@ -36,7 +36,21 @@ func (a *auditServer) WatchEvents(req *WatchEventsRequest, srv Audit_WatchEvents func (a *auditServer) RegisterFileSink(_ context.Context, req *RegisterFileSinkRequest) (resp *RegisterFileSinkResponse, err error) { var writer io.WriteCloser - if writer, err = os.OpenFile(req.TargetPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644); err != nil { + var flags int + + switch req.OpenMode { + case FileOpenMode_APPEND: + flags = os.O_CREATE | os.O_WRONLY | os.O_APPEND + default: + flags = os.O_CREATE | os.O_WRONLY | os.O_TRUNC + } + + var permissions = os.FileMode(req.Permissions) + if permissions == 0 { + permissions = 644 + } + + if writer, err = os.OpenFile(req.TargetPath, flags, permissions); err != nil { return } if err = a.eventStream.RegisterSink(sink.NewWriterSink(req.TargetPath, audit.NewEventWriter(writer))); err != nil { From a76d1d1c047f32fecb352f0b149c4a1ca9ccf5d9 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Mon, 18 Jan 2021 18:41:07 +0100 Subject: [PATCH 15/26] Cleanup in server app --- cmd/inetmock/ca.go | 2 +- cmd/inetmock/main.go | 19 +++++-------------- cmd/inetmock/serve.go | 16 ++++++++-------- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/cmd/inetmock/ca.go b/cmd/inetmock/ca.go index 1f54b47..62f2279 100644 --- a/cmd/inetmock/ca.go +++ b/cmd/inetmock/ca.go @@ -57,7 +57,7 @@ func init() { } func runGenerateCA(_ *cobra.Command, _ []string) { - logger := server.Logger().Named("generate-ca") + logger := serverApp.Logger().Named("generate-ca") logger = logger.With( zap.String(generateCACurveName, curveName), diff --git a/cmd/inetmock/main.go b/cmd/inetmock/main.go index bcec3ca..cab14d1 100644 --- a/cmd/inetmock/main.go +++ b/cmd/inetmock/main.go @@ -1,30 +1,20 @@ package main import ( - "fmt" - "gitlab.com/inetmock/inetmock/internal/app" dns "gitlab.com/inetmock/inetmock/internal/endpoint/handler/dns/mock" http "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/mock" "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/proxy" "gitlab.com/inetmock/inetmock/internal/endpoint/handler/metrics" "gitlab.com/inetmock/inetmock/internal/endpoint/handler/tls/interceptor" - "go.uber.org/zap" ) var ( - server app.App + serverApp app.App ) func main() { - logger, _ := zap.NewProduction() - defer func() { - if err := logger.Sync(); err != nil { - fmt.Println(err.Error()) - } - }() - - app.NewApp("inetmock", "INetMock is lightweight internet mock"). + serverApp = app.NewApp("inetmock", "INetMock is lightweight internet mock"). WithHandlerRegistry( http.AddHTTPMock, dns.AddDNSMock, @@ -37,6 +27,7 @@ func main() { WithHealthChecker(). WithCertStore(). WithEventStream(). - WithEndpointManager(). - MustRun() + WithEndpointManager() + + serverApp.MustRun() } diff --git a/cmd/inetmock/serve.go b/cmd/inetmock/serve.go index cff2dff..fd5dc5b 100644 --- a/cmd/inetmock/serve.go +++ b/cmd/inetmock/serve.go @@ -19,13 +19,13 @@ var ( ) func startINetMock(_ *cobra.Command, _ []string) { - rpcAPI := rpc.NewINetMockAPI(server) - logger := server.Logger().Named("inetmock").With(zap.String("command", "serve")) + rpcAPI := rpc.NewINetMockAPI(serverApp) + logger := serverApp.Logger().Named("inetmock").With(zap.String("command", "serve")) - for endpointName, endpointHandler := range server.Config().EndpointConfigs() { - handlerSubConfig := server.Config().Viper().Sub(strings.Join([]string{config.EndpointsKey, endpointName, config.OptionsKey}, ".")) + for endpointName, endpointHandler := range serverApp.Config().EndpointConfigs() { + handlerSubConfig := serverApp.Config().Viper().Sub(strings.Join([]string{config.EndpointsKey, endpointName, config.OptionsKey}, ".")) endpointHandler.Options = handlerSubConfig - if err := server.EndpointManager().CreateEndpoint(endpointName, endpointHandler); err != nil { + if err := serverApp.EndpointManager().CreateEndpoint(endpointName, endpointHandler); err != nil { logger.Warn( "error occurred while creating endpoint", zap.String("endpointName", endpointName), @@ -35,7 +35,7 @@ func startINetMock(_ *cobra.Command, _ []string) { } } - server.EndpointManager().StartEndpoints() + serverApp.EndpointManager().StartEndpoints() if err := rpcAPI.StartServer(); err != nil { logger.Error( "failed to start gRPC API", @@ -43,12 +43,12 @@ func startINetMock(_ *cobra.Command, _ []string) { ) } - <-server.Context().Done() + <-serverApp.Context().Done() logger.Info( "App context canceled - shutting down", ) rpcAPI.StopServer() - server.EndpointManager().ShutdownEndpoints() + serverApp.EndpointManager().ShutdownEndpoints() } From 66f2aab9af80b4978e97801d26b7a550e362dfc8 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Wed, 20 Jan 2021 18:39:57 +0100 Subject: [PATCH 16/26] Add commands to list sinks and to read a protocol file to JSON --- api/proto/internal/rpc/audit.proto | 9 +++ cmd/imctl/audit_file.go | 63 --------------- cmd/imctl/audit_sinks.go | 118 +++++++++++++++++++++++++++++ cmd/imctl/endpoints.go | 4 +- cmd/imctl/main.go | 21 ++--- internal/rpc/audit_server.go | 6 ++ 6 files changed, 146 insertions(+), 75 deletions(-) delete mode 100644 cmd/imctl/audit_file.go create mode 100644 cmd/imctl/audit_sinks.go diff --git a/api/proto/internal/rpc/audit.proto b/api/proto/internal/rpc/audit.proto index 7494a5c..3a1aab6 100644 --- a/api/proto/internal/rpc/audit.proto +++ b/api/proto/internal/rpc/audit.proto @@ -36,8 +36,17 @@ message RemoveFileSinkResponse { bool SinkGotRemoved = 1; } +message ListSinksRequest { + +} + +message ListSinksResponse { + repeated string sinks = 1; +} + service Audit { rpc WatchEvents (WatchEventsRequest) returns (stream inetmock.audit.EventEntity); rpc RegisterFileSink (RegisterFileSinkRequest) returns (RegisterFileSinkResponse); rpc RemoveFileSink (RemoveFileSinkRequest) returns (RemoveFileSinkResponse); + rpc ListSinks(ListSinksRequest) returns (ListSinksResponse); } \ No newline at end of file diff --git a/cmd/imctl/audit_file.go b/cmd/imctl/audit_file.go deleted file mode 100644 index 132067f..0000000 --- a/cmd/imctl/audit_file.go +++ /dev/null @@ -1,63 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - - "github.com/spf13/cobra" - "gitlab.com/inetmock/inetmock/internal/rpc" - "google.golang.org/grpc" -) - -var ( - addFileCmd = &cobra.Command{ - Use: "addFile", - Short: "subscribe events to a file", - Args: cobra.ExactArgs(1), - RunE: runAddFile, - } - - removeFileCmd = &cobra.Command{ - Use: "removeFile", - Short: "remove file subscription", - Args: cobra.ExactArgs(1), - RunE: runRemoveFile, - } - - readFileCmd = &cobra.Command{ - Use: "readFile", - Short: "reads an audit file and prints the events", - Args: cobra.ExactArgs(1), - RunE: runReadFile, - } -) - -func runAddFile(_ *cobra.Command, args []string) (err error) { - var conn *grpc.ClientConn - - if conn, err = grpc.Dial(inetMockSocketPath, grpc.WithInsecure()); err != nil { - fmt.Printf("Failed to connecto INetMock socket: %v\n", err) - os.Exit(10) - } - - auditClient := rpc.NewAuditClient(conn) - ctx, cancel := context.WithTimeout(cliApp.Context(), grpcTimeout) - defer cancel() - - _, err = auditClient.RegisterFileSink(ctx, &rpc.RegisterFileSinkRequest{TargetPath: args[0]}) - return -} - -func runRemoveFile(_ *cobra.Command, args []string) (err error) { - auditClient := rpc.NewAuditClient(conn) - ctx, cancel := context.WithTimeout(cliApp.Context(), grpcTimeout) - defer cancel() - - _, err = auditClient.RemoveFileSink(ctx, &rpc.RemoveFileSinkRequest{TargetPath: args[0]}) - return -} - -func runReadFile(_ *cobra.Command, args []string) (err error) { - return -} diff --git a/cmd/imctl/audit_sinks.go b/cmd/imctl/audit_sinks.go new file mode 100644 index 0000000..75aefb5 --- /dev/null +++ b/cmd/imctl/audit_sinks.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" + "gitlab.com/inetmock/inetmock/internal/format" + "gitlab.com/inetmock/inetmock/internal/rpc" + "gitlab.com/inetmock/inetmock/pkg/audit" +) + +var ( + listSinksCmd = &cobra.Command{ + Use: "list", + Aliases: []string{"ls", "dir"}, + Short: "List all subscribed sinks", + RunE: runListSinks, + } + addFileCmd = &cobra.Command{ + Use: "add-file", + Aliases: []string{"add"}, + Short: "subscribe events to a file", + Args: cobra.ExactArgs(1), + RunE: runAddFile, + } + + removeFileCmd = &cobra.Command{ + Use: "remove-file", + Aliases: []string{"rm", "del"}, + Short: "remove file subscription", + Args: cobra.ExactArgs(1), + RunE: runRemoveFile, + } + + readFileCmd = &cobra.Command{ + Use: "read-file", + Aliases: []string{"cat"}, + Short: "reads an audit file and prints the events", + Args: cobra.ExactArgs(1), + RunE: runReadFile, + } +) + +type printableSink struct { + Name string +} + +func runListSinks(*cobra.Command, []string) (err error) { + auditClient := rpc.NewAuditClient(conn) + ctx, cancel := context.WithTimeout(cliApp.Context(), grpcTimeout) + defer cancel() + + var resp *rpc.ListSinksResponse + if resp, err = auditClient.ListSinks(ctx, &rpc.ListSinksRequest{}); err != nil { + return + } + + var sinks []printableSink + for _, s := range resp.Sinks { + sinks = append(sinks, printableSink{Name: s}) + } + + writer := format.Writer(outputFormat, os.Stdout) + err = writer.Write(sinks) + return +} + +func runAddFile(_ *cobra.Command, args []string) (err error) { + auditClient := rpc.NewAuditClient(conn) + ctx, cancel := context.WithTimeout(cliApp.Context(), grpcTimeout) + defer cancel() + + _, err = auditClient.RegisterFileSink(ctx, &rpc.RegisterFileSinkRequest{TargetPath: args[0]}) + return +} + +func runRemoveFile(_ *cobra.Command, args []string) (err error) { + auditClient := rpc.NewAuditClient(conn) + ctx, cancel := context.WithTimeout(cliApp.Context(), grpcTimeout) + defer cancel() + + _, err = auditClient.RemoveFileSink(ctx, &rpc.RemoveFileSinkRequest{TargetPath: args[0]}) + return +} + +func runReadFile(_ *cobra.Command, args []string) (err error) { + if len(args) != 1 { + return errors.New("expected only 1 argument") + } + + var reader io.ReadCloser + if reader, err = os.Open(args[0]); err != nil { + return + } + + eventReader := audit.NewEventReader(reader) + var ev audit.Event + + for err == nil { + if ev, err = eventReader.Read(); err == nil { + var jsonBytes []byte + if jsonBytes, err = json.Marshal(ev); err == nil { + fmt.Println(string(jsonBytes)) + } + } + } + + if errors.Is(err, io.EOF) { + err = nil + } + + return +} diff --git a/cmd/imctl/endpoints.go b/cmd/imctl/endpoints.go index 592ef9b..622dd4d 100644 --- a/cmd/imctl/endpoints.go +++ b/cmd/imctl/endpoints.go @@ -25,7 +25,7 @@ var ( ) type printableEndpoint struct { - Id string + ID string Name string Handler string ListenAddress string @@ -34,7 +34,7 @@ type printableEndpoint struct { func fromEndpoint(ep *rpc.Endpoint) *printableEndpoint { return &printableEndpoint{ - Id: ep.Id, + ID: ep.Id, Name: ep.Name, Handler: ep.Handler, ListenAddress: ep.ListenAddress, diff --git a/cmd/imctl/main.go b/cmd/imctl/main.go index 1ca965f..93a0803 100644 --- a/cmd/imctl/main.go +++ b/cmd/imctl/main.go @@ -2,6 +2,8 @@ package main import ( "context" + "fmt" + "os" "os/user" "time" @@ -20,7 +22,6 @@ var ( ) func main() { - endpointsCmd.AddCommand(getEndpoints) handlerCmd.AddCommand(getHandlersCmd) healthCmd.AddCommand(generalHealthCmd, containerHealthCmd) @@ -29,7 +30,8 @@ func main() { WithCommands(endpointsCmd, handlerCmd, healthCmd, auditCmd). WithInitTasks(func(_ *cobra.Command, _ []string) (err error) { return initGRPCConnection() - }) + }). + WithLogger() cliApp.RootCommand().PersistentFlags().StringVar(&inetMockSocketPath, "socket-path", "unix:///var/run/inetmock.sock", "Path to the INetMock socket file") cliApp.RootCommand().PersistentFlags().StringVarP(&outputFormat, "format", "f", "table", "Output format to use. Possible values: table, json, yaml") @@ -42,16 +44,15 @@ func main() { currentUser = uuid.New().String() } - watchEventsCmd.PersistentFlags().StringVar( - &listenerName, - "listener-name", - currentUser, - "set listener name - defaults to the current username, if the user cannot be determined a random UUID will be used", - ) - auditCmd.AddCommand(watchEventsCmd, addFileCmd, removeFileCmd) + hostname := "." + if hn, err := os.Hostname(); err == nil { + hostname = hn + } + + watchEventsCmd.PersistentFlags().StringVar(&listenerName, "listener-name", fmt.Sprintf("%s\\%s is watching", hostname, currentUser), "set listener name - defaults to the current username, if the user cannot be determined a random UUID will be used") + auditCmd.AddCommand(listSinksCmd, watchEventsCmd, addFileCmd, removeFileCmd, readFileCmd) cliApp.MustRun() - } func initGRPCConnection() (err error) { diff --git a/internal/rpc/audit_server.go b/internal/rpc/audit_server.go index 96bdc63..0fcfa8a 100644 --- a/internal/rpc/audit_server.go +++ b/internal/rpc/audit_server.go @@ -17,6 +17,12 @@ type auditServer struct { eventStream audit.EventStream } +func (a *auditServer) ListSinks(context.Context, *ListSinksRequest) (*ListSinksResponse, error) { + return &ListSinksResponse{ + Sinks: a.eventStream.Sinks(), + }, nil +} + func (a *auditServer) WatchEvents(req *WatchEventsRequest, srv Audit_WatchEventsServer) (err error) { a.logger.Info("watcher attached", zap.String("name", req.WatcherName)) err = a.eventStream.RegisterSink(sink.NewGRPCSink(srv.Context(), req.WatcherName, func(ev audit.Event) { From af0a7a2375a96b632dfd3d1b9bab0945a0a77cb4 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Wed, 20 Jan 2021 18:43:00 +0100 Subject: [PATCH 17/26] Capture source and destination addresses as byte arrays - update necessary tests This removes a lot of complexity because IPv4 and IPv6 addresses can be handled the same way. To distinguish between them it's enough to take their length into account. Parsing should be straight forward in any language. --- api/proto/pkg/audit/event_entity.proto | 22 ++++----------- pkg/audit/event.go | 38 +++----------------------- pkg/audit/ip_conversion.go | 33 ---------------------- pkg/audit/reader_test.go | 10 ++++--- 4 files changed, 16 insertions(+), 87 deletions(-) delete mode 100644 pkg/audit/ip_conversion.go diff --git a/api/proto/pkg/audit/event_entity.proto b/api/proto/pkg/audit/event_entity.proto index 2939193..00492af 100644 --- a/api/proto/pkg/audit/event_entity.proto +++ b/api/proto/pkg/audit/event_entity.proto @@ -42,20 +42,10 @@ message EventEntity { google.protobuf.Timestamp timestamp = 2; TransportProtocol transport = 3; AppProtocol application = 4; - - oneof sourceIP { - uint32 sourceIPv4 = 5; - uint64 sourceIPv6 = 6; - } - - oneof destinationIP { - uint32 destinationIPv4 = 7; - uint64 destinationIPv6 = 8; - } - - uint32 sourcePort = 9; - uint32 destinationPort = 10; - - TLSDetailsEntity tls = 11; - google.protobuf.Any protocolDetails = 12; + bytes sourceIP = 5; + bytes destinationIP = 6; + uint32 sourcePort = 7; + uint32 destinationPort = 8; + TLSDetailsEntity tls = 9; + google.protobuf.Any protocolDetails = 10; } \ No newline at end of file diff --git a/pkg/audit/event.go b/pkg/audit/event.go index 5e2f29d..0e7576c 100644 --- a/pkg/audit/event.go +++ b/pkg/audit/event.go @@ -30,20 +30,6 @@ type Event struct { } func (e *Event) ProtoMessage() *EventEntity { - var sourceIP isEventEntity_SourceIP - if ipv4 := e.SourceIP.To4(); ipv4 != nil { - sourceIP = &EventEntity_SourceIPv4{SourceIPv4: ipv4ToUint32(ipv4)} - } else { - sourceIP = &EventEntity_SourceIPv6{SourceIPv6: ipv6ToBytes(e.SourceIP)} - } - - var destinationIP isEventEntity_DestinationIP - if ipv4 := e.DestinationIP.To4(); ipv4 != nil { - destinationIP = &EventEntity_DestinationIPv4{DestinationIPv4: ipv4ToUint32(ipv4)} - } else { - destinationIP = &EventEntity_DestinationIPv6{DestinationIPv6: ipv6ToBytes(e.DestinationIP)} - } - var tlsDetails *TLSDetailsEntity = nil if e.TLS != nil { tlsDetails = e.TLS.ProtoMessage() @@ -61,8 +47,8 @@ func (e *Event) ProtoMessage() *EventEntity { Timestamp: timestamppb.New(e.Timestamp), Transport: e.Transport, Application: e.Application, - SourceIP: sourceIP, - DestinationIP: destinationIP, + SourceIP: e.SourceIP, + DestinationIP: e.DestinationIP, SourcePort: uint32(e.SourcePort), DestinationPort: uint32(e.DestinationPort), Tls: tlsDetails, @@ -91,29 +77,13 @@ func (e *Event) SetDestinationIPFromAddr(localAddr net.Addr) { } func NewEventFromProto(msg *EventEntity) (ev Event) { - var sourceIP net.IP - switch ip := msg.GetSourceIP().(type) { - case *EventEntity_SourceIPv4: - sourceIP = uint32ToIP(ip.SourceIPv4) - case *EventEntity_SourceIPv6: - sourceIP = uint64ToIP(ip.SourceIPv6) - } - - var destinationIP net.IP - switch ip := msg.GetDestinationIP().(type) { - case *EventEntity_DestinationIPv4: - destinationIP = uint32ToIP(ip.DestinationIPv4) - case *EventEntity_DestinationIPv6: - destinationIP = uint64ToIP(ip.DestinationIPv6) - } - ev = Event{ ID: msg.GetId(), Timestamp: msg.GetTimestamp().AsTime(), Transport: msg.GetTransport(), Application: msg.GetApplication(), - SourceIP: sourceIP, - DestinationIP: destinationIP, + SourceIP: msg.SourceIP, + DestinationIP: msg.DestinationIP, SourcePort: uint16(msg.GetSourcePort()), DestinationPort: uint16(msg.GetDestinationPort()), ProtocolDetails: guessDetailsFromApp(msg.GetProtocolDetails()), diff --git a/pkg/audit/ip_conversion.go b/pkg/audit/ip_conversion.go deleted file mode 100644 index 7385eb3..0000000 --- a/pkg/audit/ip_conversion.go +++ /dev/null @@ -1,33 +0,0 @@ -package audit - -import ( - "encoding/binary" - "math/big" - "net" -) - -func ipv4ToUint32(ip net.IP) uint32 { - if len(ip) == 16 { - return binary.BigEndian.Uint32(ip[12:16]) - } - return binary.BigEndian.Uint32(ip) -} - -func ipv6ToBytes(ip net.IP) uint64 { - ipv6 := big.NewInt(0) - ipv6.SetBytes(ip) - return ipv6.Uint64() -} - -func uint32ToIP(i uint32) (ip net.IP) { - buf := make([]byte, 4) - binary.BigEndian.PutUint32(buf, i) - ip = buf - ip = ip.To4() - return -} - -func uint64ToIP(i uint64) (ip net.IP) { - ip = big.NewInt(int64(i)).FillBytes(make([]byte, 16)) - return -} diff --git a/pkg/audit/reader_test.go b/pkg/audit/reader_test.go index e65ec7f..94c2e32 100644 --- a/pkg/audit/reader_test.go +++ b/pkg/audit/reader_test.go @@ -13,11 +13,13 @@ import ( var ( //nolint:lll - httpPayloadBytesLittleEndian = `dd000000120b088092b8c398feffffff011801200248d8fc0150505a3308041224544c535f45434448455f45434453415f574954485f4145535f3235365f4342435f5348411a096c6f63616c686f73746282010a34747970652e676f6f676c65617069732e636f6d2f696e65746d6f636b2e61756469742e4854545044657461696c73456e74697479124a12096c6f63616c686f73741a15687474703a2f2f6c6f63616c686f73742f6173646622084854545020312e312a1c0a0641636365707412120a106170706c69636174696f6e2f6a736f6e28818080f80738818080f807` + httpPayloadBytesLittleEndian = `dd000000120b088092b8c398feffffff01180120022a047f00000132047f00000138d8fc0140504a3308041224544c535f45434448455f45434453415f574954485f4145535f3235365f4342435f5348411a096c6f63616c686f73745282010a34747970652e676f6f676c65617069732e636f6d2f696e65746d6f636b2e61756469742e4854545044657461696c73456e74697479124a12096c6f63616c686f73741a15687474703a2f2f6c6f63616c686f73742f6173646622084854545020312e312a1c0a0641636365707412120a106170706c69636174696f6e2f6a736f6e` //nolint:lll - httpPayloadBytesBigEndian = `000000dd120b088092b8c398feffffff011801200248d8fc0150505a3308041224544c535f45434448455f45434453415f574954485f4145535f3235365f4342435f5348411a096c6f63616c686f73746282010a34747970652e676f6f676c65617069732e636f6d2f696e65746d6f636b2e61756469742e4854545044657461696c73456e74697479124a12096c6f63616c686f73741a15687474703a2f2f6c6f63616c686f73742f6173646622084854545020312e312a1c0a0641636365707412120a106170706c69636174696f6e2f6a736f6e28818080f80738818080f807` - dnsPayloadBytesLittleEndian = `1b000000120b088092b8c398feffffff011801200148d8fc01505030014001` - dnsPayloadBytesBigEndian = `0000001b120b088092b8c398feffffff011801200148d8fc01505030014001` + httpPayloadBytesBigEndian = `000000dd120b088092b8c398feffffff01180120022a047f00000132047f00000138d8fc0140504a3308041224544c535f45434448455f45434453415f574954485f4145535f3235365f4342435f5348411a096c6f63616c686f73745282010a34747970652e676f6f676c65617069732e636f6d2f696e65746d6f636b2e61756469742e4854545044657461696c73456e74697479124a12096c6f63616c686f73741a15687474703a2f2f6c6f63616c686f73742f6173646622084854545020312e312a1c0a0641636365707412120a106170706c69636174696f6e2f6a736f6e` + //nolint:lll + dnsPayloadBytesLittleEndian = `3b000000120b088092b8c398feffffff01180120012a100000000000000000000000000000000132100000000000000000000000000000000138d8fc014050` + //nolint:lll + dnsPayloadBytesBigEndian = `0000003b120b088092b8c398feffffff01180120012a100000000000000000000000000000000132100000000000000000000000000000000138d8fc014050` ) func mustDecodeHex(hexBytes string) io.Reader { From 03f3ae4173e0403312637c86cd973e0d8763c7d5 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Wed, 20 Jan 2021 18:43:28 +0100 Subject: [PATCH 18/26] Enable gRPC reflection to support gRPCurl and gRPCui --- internal/rpc/grpc_api.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/rpc/grpc_api.go b/internal/rpc/grpc_api.go index b5f0f65..b523d7f 100644 --- a/internal/rpc/grpc_api.go +++ b/internal/rpc/grpc_api.go @@ -9,6 +9,7 @@ import ( "gitlab.com/inetmock/inetmock/pkg/logging" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/reflection" ) type INetMockAPI interface { @@ -57,6 +58,8 @@ func (i *inetmockAPI) StartServer() (err error) { eventStream: i.app.EventStream(), }) + reflection.Register(i.server) + go i.startServerAsync(lis) return } From 63a6516d99e76e9257ea1ee9d57ebea577aa8821 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Wed, 20 Jan 2021 19:03:05 +0100 Subject: [PATCH 19/26] Added audit stream to HTTP proxy --- internal/endpoint/handler/http/audit.go | 37 +++++++++++++++++++ .../handler/http/{mock => }/conn_context.go | 10 ++--- .../endpoint/handler/http/mock/handler.go | 3 +- .../handler/http/mock/regex_router.go | 25 +------------ .../endpoint/handler/http/proxy/handler.go | 13 +++++-- .../handler/http/proxy/proxy_handler.go | 17 +++------ pkg/audit/event.go | 3 ++ 7 files changed, 65 insertions(+), 43 deletions(-) create mode 100644 internal/endpoint/handler/http/audit.go rename internal/endpoint/handler/http/{mock => }/conn_context.go (58%) diff --git a/internal/endpoint/handler/http/audit.go b/internal/endpoint/handler/http/audit.go new file mode 100644 index 0000000..9d92817 --- /dev/null +++ b/internal/endpoint/handler/http/audit.go @@ -0,0 +1,37 @@ +package http + +import ( + "net/http" + + "gitlab.com/inetmock/inetmock/pkg/audit" + "gitlab.com/inetmock/inetmock/pkg/audit/details" +) + +func EventFromRequest(request *http.Request, app audit.AppProtocol) audit.Event { + httpDetails := details.HTTP{ + Method: request.Method, + Host: request.Host, + URI: request.RequestURI, + Proto: request.Proto, + Headers: request.Header, + } + + ev := audit.Event{ + Transport: audit.TransportProtocol_TCP, + Application: app, + ProtocolDetails: httpDetails, + } + + if request.TLS != nil { + ev.TLS = &audit.TLSDetails{ + Version: request.TLS.Version, + CipherSuite: request.TLS.CipherSuite, + ServerName: request.TLS.ServerName, + } + } + + ev.SetDestinationIPFromAddr(localAddr(request.Context())) + ev.SetSourceIPFromAddr(remoteAddr(request.Context())) + + return ev +} diff --git a/internal/endpoint/handler/http/mock/conn_context.go b/internal/endpoint/handler/http/conn_context.go similarity index 58% rename from internal/endpoint/handler/http/mock/conn_context.go rename to internal/endpoint/handler/http/conn_context.go index 502d104..3f9c6da 100644 --- a/internal/endpoint/handler/http/mock/conn_context.go +++ b/internal/endpoint/handler/http/conn_context.go @@ -1,4 +1,4 @@ -package mock +package http import ( "context" @@ -8,8 +8,8 @@ import ( type httpContextKey string const ( - remoteAddrKey httpContextKey = "RemoteAddr" - localAddrKey httpContextKey = "LocalAddr" + remoteAddrKey httpContextKey = "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/context/remoteAddr" + localAddrKey httpContextKey = "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http/context/localAddr" ) func StoreConnPropertiesInContext(ctx context.Context, c net.Conn) context.Context { @@ -18,7 +18,7 @@ func StoreConnPropertiesInContext(ctx context.Context, c net.Conn) context.Conte return ctx } -func LocalAddr(ctx context.Context) net.Addr { +func localAddr(ctx context.Context) net.Addr { val := ctx.Value(localAddrKey) if val == nil { return nil @@ -26,7 +26,7 @@ func LocalAddr(ctx context.Context) net.Addr { return val.(net.Addr) } -func RemoteAddr(ctx context.Context) net.Addr { +func remoteAddr(ctx context.Context) net.Addr { val := ctx.Value(remoteAddrKey) if val == nil { return nil diff --git a/internal/endpoint/handler/http/mock/handler.go b/internal/endpoint/handler/http/mock/handler.go index 486b22e..f4ad94d 100644 --- a/internal/endpoint/handler/http/mock/handler.go +++ b/internal/endpoint/handler/http/mock/handler.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" + imHttp "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http" "gitlab.com/inetmock/inetmock/pkg/api" "gitlab.com/inetmock/inetmock/pkg/config" "gitlab.com/inetmock/inetmock/pkg/logging" @@ -46,7 +47,7 @@ func (p *httpHandler) Start(ctx api.PluginContext, config config.HandlerConfig) p.server = &http.Server{ Addr: config.ListenAddr(), Handler: router, - ConnContext: StoreConnPropertiesInContext, + ConnContext: imHttp.StoreConnPropertiesInContext, } for _, rule := range options.Rules { diff --git a/internal/endpoint/handler/http/mock/regex_router.go b/internal/endpoint/handler/http/mock/regex_router.go index 4e96329..28f9d8b 100644 --- a/internal/endpoint/handler/http/mock/regex_router.go +++ b/internal/endpoint/handler/http/mock/regex_router.go @@ -5,8 +5,8 @@ import ( "strconv" "github.com/prometheus/client_golang/prometheus" + imHttp "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http" "gitlab.com/inetmock/inetmock/pkg/audit" - details "gitlab.com/inetmock/inetmock/pkg/audit/details" "gitlab.com/inetmock/inetmock/pkg/logging" "go.uber.org/zap" ) @@ -63,27 +63,6 @@ type emittingFileHandler struct { } func (f emittingFileHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) { - f.emitter.Emit(eventFromRequest(request)) + f.emitter.Emit(imHttp.EventFromRequest(request, audit.AppProtocol_HTTP)) http.ServeFile(writer, request, f.targetPath) } - -func eventFromRequest(request *http.Request) audit.Event { - httpDetails := details.HTTP{ - Method: request.Method, - Host: request.Host, - URI: request.RequestURI, - Proto: request.Proto, - Headers: request.Header, - } - - ev := audit.Event{ - Transport: audit.TransportProtocol_TCP, - Application: audit.AppProtocol_HTTP, - ProtocolDetails: httpDetails, - } - - ev.SetDestinationIPFromAddr(LocalAddr(request.Context())) - ev.SetSourceIPFromAddr(RemoteAddr(request.Context())) - - return ev -} diff --git a/internal/endpoint/handler/http/proxy/handler.go b/internal/endpoint/handler/http/proxy/handler.go index 3eef03a..08c12d5 100644 --- a/internal/endpoint/handler/http/proxy/handler.go +++ b/internal/endpoint/handler/http/proxy/handler.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" + imHttp "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http" "gitlab.com/inetmock/inetmock/pkg/api" "gitlab.com/inetmock/inetmock/pkg/config" "gitlab.com/inetmock/inetmock/pkg/logging" @@ -29,7 +30,11 @@ func (h *httpProxy) Start(ctx api.PluginContext, cfg config.HandlerConfig) (err return } listenAddr := cfg.ListenAddr() - h.server = &http.Server{Addr: listenAddr, Handler: h.proxy} + h.server = &http.Server{ + Addr: listenAddr, + Handler: h.proxy, + ConnContext: imHttp.StoreConnPropertiesInContext, + } h.logger = h.logger.With( zap.String("handler_name", cfg.HandlerName), zap.String("address", listenAddr), @@ -41,16 +46,18 @@ func (h *httpProxy) Start(ctx api.PluginContext, cfg config.HandlerConfig) (err handlerName: cfg.HandlerName, options: opts, logger: h.logger, + emitter: ctx.Audit(), } - proxyHttpsHandler := &proxyHttpsHandler{ + proxyHTTPSHandler := &proxyHttpsHandler{ handlerName: cfg.HandlerName, tlsConfig: tlsConfig, logger: h.logger, + emitter: ctx.Audit(), } h.proxy.OnRequest().Do(proxyHandler) - h.proxy.OnRequest().HandleConnect(proxyHttpsHandler) + h.proxy.OnRequest().HandleConnect(proxyHTTPSHandler) go h.startProxy() return } diff --git a/internal/endpoint/handler/http/proxy/proxy_handler.go b/internal/endpoint/handler/http/proxy/proxy_handler.go index d7af5b8..6bdf26e 100644 --- a/internal/endpoint/handler/http/proxy/proxy_handler.go +++ b/internal/endpoint/handler/http/proxy/proxy_handler.go @@ -1,12 +1,13 @@ package proxy import ( - "context" "crypto/tls" "net/http" "net/url" "github.com/prometheus/client_golang/prometheus" + imHttp "gitlab.com/inetmock/inetmock/internal/endpoint/handler/http" + "gitlab.com/inetmock/inetmock/pkg/audit" "gitlab.com/inetmock/inetmock/pkg/logging" "go.uber.org/zap" "gopkg.in/elazarl/goproxy.v1" @@ -16,12 +17,14 @@ type proxyHttpHandler struct { handlerName string options httpProxyOptions logger logging.Logger + emitter audit.Emitter } type proxyHttpsHandler struct { handlerName string tlsConfig *tls.Config logger logging.Logger + emitter audit.Emitter } func (p *proxyHttpsHandler) HandleConnect(req string, _ *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) { @@ -45,15 +48,7 @@ func (p *proxyHttpHandler) Handle(req *http.Request, ctx *goproxy.ProxyCtx) (ret totalRequestCounter.WithLabelValues(p.handlerName).Inc() retReq = req - p.logger.Info( - "Handling request", - zap.String("source", req.RemoteAddr), - zap.String("host", req.Host), - zap.String("method", req.Method), - zap.String("protocol", req.Proto), - zap.String("path", req.RequestURI), - zap.Reflect("headers", req.Header), - ) + p.emitter.Emit(imHttp.EventFromRequest(req, audit.AppProtocol_HTTP_PROXY)) var err error if resp, err = ctx.RoundTrip(p.redirectHTTPRequest(req)); err != nil { @@ -95,7 +90,7 @@ func (p proxyHttpHandler) redirectHTTPRequest(originalRequest *http.Request) (re MultipartForm: originalRequest.MultipartForm, Trailer: originalRequest.Trailer, } - redirectReq = redirectReq.WithContext(context.Background()) + redirectReq = redirectReq.WithContext(originalRequest.Context()) return } diff --git a/pkg/audit/event.go b/pkg/audit/event.go index 0e7576c..dc9f696 100644 --- a/pkg/audit/event.go +++ b/pkg/audit/event.go @@ -93,6 +93,9 @@ func NewEventFromProto(msg *EventEntity) (ev Event) { } func parseIPPortFromAddr(addr net.Addr) (ip net.IP, port uint16) { + if addr == nil { + return + } switch a := addr.(type) { case *net.TCPAddr: return a.IP, uint16(a.Port) From 6d2737b50134e52b58c5077e03d913ed3c1fa128 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Tue, 26 Jan 2021 18:42:07 +0100 Subject: [PATCH 20/26] Move HTTPS handling to http_mock handler --- assets/demoCA/ca.key | 5 -- assets/demoCA/ca.pem | 12 ----- cmd/inetmock/ca.go | 26 +--------- config.yaml | 19 ++++---- internal/app/app.go | 48 +++++++++++++++---- internal/endpoint/endpoint_manager.go | 2 - internal/endpoint/handler/http/audit.go | 5 +- .../endpoint/handler/http/mock/handler.go | 21 ++++++-- .../handler/http/mock/protocol_options.go | 4 ++ .../http/mock/protocol_options_test.go | 28 +++++++++++ pkg/audit/event_stream_test.go | 4 +- pkg/audit/sink/log_sink.go | 5 +- pkg/audit/sink/writer_sink_test.go | 4 +- pkg/audit/tls_details.go | 38 ++++++--------- 14 files changed, 124 insertions(+), 97 deletions(-) delete mode 100644 assets/demoCA/ca.key delete mode 100644 assets/demoCA/ca.pem diff --git a/assets/demoCA/ca.key b/assets/demoCA/ca.key deleted file mode 100644 index b3aa03e..0000000 --- a/assets/demoCA/ca.key +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgTTz25fFLS2WO4hXD -162B059HEe+MAQtV4iGXf7HfKCihRANCAAT3D181Tzrz6i9Mx75pmyAsg+itojO9 -sHXZSswmfsh46IVK46m0hXNHgPvD2WYW5m1PHvRl3B0vDo/2Y6sOU/Q9 ------END PRIVATE KEY----- diff --git a/assets/demoCA/ca.pem b/assets/demoCA/ca.pem deleted file mode 100644 index c603b9c..0000000 --- a/assets/demoCA/ca.pem +++ /dev/null @@ -1,12 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIB3DCCAYKgAwIBAgIQHQIFIEcNZjsDP+wDtGPMXzAKBggqhkjOPQQDAjBOMRAw -DgYDVQQGEwdnZXJtYW55MREwDwYDVQQHEwhEb3J0bXVuZDERMA8GA1UEChMISU5l -dE1vY2sxFDASBgNVBAMTC0lOZXRNb2NrIENBMB4XDTIwMDYxNTEwNTEzNloXDTIw -MDYxNTEwNTEzNlowTjEQMA4GA1UEBhMHZ2VybWFueTERMA8GA1UEBxMIRG9ydG11 -bmQxETAPBgNVBAoTCElOZXRNb2NrMRQwEgYDVQQDEwtJTmV0TW9jayBDQTBZMBMG -ByqGSM49AgEGCCqGSM49AwEHA0IABPcPXzVPOvPqL0zHvmmbICyD6K2iM72wddlK -zCZ+yHjohUrjqbSFc0eA+8PZZhbmbU8e9GXcHS8Oj/Zjqw5T9D2jQjBAMA4GA1Ud -DwEB/wQEAwIChDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDwYDVR0T -AQH/BAUwAwEB/zAKBggqhkjOPQQDAgNIADBFAiBecJsOL7ej0kCkWOnoQJpW3JuY -KQIxQBT+XXPKEJj14AIhANG4twTloC3amz8Y7Zn3DVtvjXlTgg8YwjBFG+JioQOe ------END CERTIFICATE----- diff --git a/cmd/inetmock/ca.go b/cmd/inetmock/ca.go index 62f2279..b28ad0f 100644 --- a/cmd/inetmock/ca.go +++ b/cmd/inetmock/ca.go @@ -8,7 +8,6 @@ import ( "github.com/spf13/cobra" "gitlab.com/inetmock/inetmock/pkg/cert" "gitlab.com/inetmock/inetmock/pkg/config" - "gitlab.com/inetmock/inetmock/pkg/logging" "go.uber.org/zap" ) @@ -34,6 +33,7 @@ var ( certOutPath, curveName string ) +//nolint:lll func init() { generateCaCmd = &cobra.Command{ Use: "generate-ca", @@ -50,7 +50,7 @@ func init() { generateCaCmd.Flags().StringSliceVar(&caCertOptions.Locality, generateCaLocalityName, nil, "Locality information to append to certificate") generateCaCmd.Flags().StringSliceVar(&caCertOptions.StreetAddress, generateCaStreetAddressName, nil, "Street address information to append to certificate") generateCaCmd.Flags().StringSliceVar(&caCertOptions.PostalCode, generateCaPostalCodeName, nil, "Postal code information to append to certificate") - generateCaCmd.Flags().StringVar(&certOutPath, generateCACertOutPath, "", "Path where CA files should be stored") + generateCaCmd.Flags().StringVar(&certOutPath, generateCACertOutPath, "", "Path where CA files should be stored") generateCaCmd.Flags().StringVar(&curveName, generateCACurveName, "", "Name of the curve to use, if empty ED25519 is used, other valid values are [P224, P256,P384,P521]") generateCaCmd.Flags().DurationVar(¬Before, generateCANotBeforeRelative, 17520*time.Hour, "Relative time value since when in the past the CA certificate should be valid. The value has a time unit, the greatest time unit is h for hour.") generateCaCmd.Flags().DurationVar(¬After, generateCANotAfterRelative, 17520*time.Hour, "Relative time value until when in the future the CA certificate should be valid. The value has a time unit, the greatest time unit is h for hour.") @@ -108,25 +108,3 @@ func runGenerateCA(_ *cobra.Command, _ []string) { } logger.Info("completed certificate generation") } - -func getDurationFlag(cmd *cobra.Command, flagName string, logger logging.Logger) (val time.Duration, err error) { - if val, err = cmd.Flags().GetDuration(flagName); err != nil { - logger.Error( - "failed to parse parse flag", - zap.String("flag", flagName), - zap.Error(err), - ) - } - return -} - -func getStringFlag(cmd *cobra.Command, flagName string, logger logging.Logger) (val string, err error) { - if val, err = cmd.Flags().GetString(flagName); err != nil { - logger.Error( - "failed to parse parse flag", - zap.String("flag", flagName), - zap.Error(err), - ) - } - return -} diff --git a/config.yaml b/config.yaml index c71558e..7ce1cdc 100644 --- a/config.yaml +++ b/config.yaml @@ -69,6 +69,15 @@ endpoints: - 8080 options: <<: *httpResponseRules + https: + handler: http_mock + listenAddress: 0.0.0.0 + ports: + - 443 + - 8443 + options: + tls: true + <<: *httpResponseRules proxy: handler: http_proxy listenAddress: 0.0.0.0 @@ -78,16 +87,6 @@ endpoints: target: ipAddress: 127.0.0.1 port: 80 - httpsDowngrade: - handler: tls_interceptor - listenAddress: 0.0.0.0 - ports: - - 443 - - 8443 - options: - target: - ipAddress: 127.0.0.1 - port: 80 plainDns: handler: dns_mock listenAddress: 0.0.0.0 diff --git a/internal/app/app.go b/internal/app/app.go index 1a8b6db..48c137b 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -108,35 +108,67 @@ func (a *app) MustRun() { } func (a *app) Logger() logging.Logger { - return a.ctx.Value(loggerKey).(logging.Logger) + val := a.ctx.Value(loggerKey) + if val == nil { + return nil + } + return val.(logging.Logger) } func (a *app) Config() config.Config { - return a.ctx.Value(configKey).(config.Config) + val := a.ctx.Value(configKey) + if val == nil { + return nil + } + return val.(config.Config) } func (a *app) CertStore() cert.Store { - return a.ctx.Value(certStoreKey).(cert.Store) + val := a.ctx.Value(certStoreKey) + if val == nil { + return nil + } + return val.(cert.Store) } func (a *app) Checker() health.Checker { - return a.ctx.Value(healthCheckerKey).(health.Checker) + val := a.ctx.Value(healthCheckerKey) + if val == nil { + return nil + } + return val.(health.Checker) } func (a *app) EndpointManager() endpoint.EndpointManager { - return a.ctx.Value(endpointManagerKey).(endpoint.EndpointManager) + val := a.ctx.Value(endpointManagerKey) + if val == nil { + return nil + } + return val.(endpoint.EndpointManager) } func (a *app) Audit() audit.Emitter { - return a.ctx.Value(eventStreamKey).(audit.Emitter) + val := a.ctx.Value(eventStreamKey) + if val == nil { + return nil + } + return val.(audit.Emitter) } func (a *app) EventStream() audit.EventStream { - return a.ctx.Value(eventStreamKey).(audit.EventStream) + val := a.ctx.Value(eventStreamKey) + if val == nil { + return nil + } + return val.(audit.EventStream) } func (a *app) HandlerRegistry() api.HandlerRegistry { - return a.ctx.Value(handlerRegistryKey).(api.HandlerRegistry) + val := a.ctx.Value(handlerRegistryKey) + if val == nil { + return nil + } + return val.(api.HandlerRegistry) } func (a *app) Context() context.Context { diff --git a/internal/endpoint/endpoint_manager.go b/internal/endpoint/endpoint_manager.go index 2da4be2..174a9d3 100644 --- a/internal/endpoint/endpoint_manager.go +++ b/internal/endpoint/endpoint_manager.go @@ -149,8 +149,6 @@ func startEndpoint(ep Endpoint, ctx api.PluginContext, logger logging.Logger) (s success = false } - close(startSuccessful) - return } diff --git a/internal/endpoint/handler/http/audit.go b/internal/endpoint/handler/http/audit.go index 9d92817..3a8d993 100644 --- a/internal/endpoint/handler/http/audit.go +++ b/internal/endpoint/handler/http/audit.go @@ -1,6 +1,7 @@ package http import ( + "crypto/tls" "net/http" "gitlab.com/inetmock/inetmock/pkg/audit" @@ -24,8 +25,8 @@ func EventFromRequest(request *http.Request, app audit.AppProtocol) audit.Event if request.TLS != nil { ev.TLS = &audit.TLSDetails{ - Version: request.TLS.Version, - CipherSuite: request.TLS.CipherSuite, + Version: audit.TLSVersionToEntity(request.TLS.Version).String(), + CipherSuite: tls.CipherSuiteName(request.TLS.CipherSuite), ServerName: request.TLS.ServerName, } } diff --git a/internal/endpoint/handler/http/mock/handler.go b/internal/endpoint/handler/http/mock/handler.go index f4ad94d..a4a8805 100644 --- a/internal/endpoint/handler/http/mock/handler.go +++ b/internal/endpoint/handler/http/mock/handler.go @@ -2,6 +2,7 @@ package mock import ( "context" + "crypto/tls" "errors" "fmt" "net/http" @@ -50,11 +51,16 @@ func (p *httpHandler) Start(ctx api.PluginContext, config config.HandlerConfig) ConnContext: imHttp.StoreConnPropertiesInContext, } + if options.TLS { + p.server.TLSConfig = ctx.CertStore().TLSConfig() + p.server.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) + } + for _, rule := range options.Rules { router.setupRoute(rule) } - go p.startServer() + go p.startServer(options.TLS) return } @@ -73,8 +79,17 @@ func (p *httpHandler) Shutdown(ctx context.Context) (err error) { return } -func (p *httpHandler) startServer() { - if err := p.server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { +func (p *httpHandler) startServer(tls bool) { + var listen func() error + if tls { + listen = func() error { + return p.server.ListenAndServeTLS("", "") + } + } else { + listen = p.server.ListenAndServe + } + + if err := listen(); err != nil && !errors.Is(err, http.ErrServerClosed) { p.logger.Error( "failed to start http listener", zap.Error(err), diff --git a/internal/endpoint/handler/http/mock/protocol_options.go b/internal/endpoint/handler/http/mock/protocol_options.go index cca7a4d..0a6099a 100644 --- a/internal/endpoint/handler/http/mock/protocol_options.go +++ b/internal/endpoint/handler/http/mock/protocol_options.go @@ -50,6 +50,7 @@ func (tr targetRule) Response() string { } type httpOptions struct { + TLS bool Rules []targetRule } @@ -62,6 +63,7 @@ func loadFromConfig(config *viper.Viper) (options httpOptions, err error) { } tmpRules := struct { + TLS bool Rules []tmpCfg }{} @@ -69,6 +71,8 @@ func loadFromConfig(config *viper.Viper) (options httpOptions, err error) { return } + options.TLS = tmpRules.TLS + for _, i := range tmpRules.Rules { var rulePattern *regexp.Regexp var matchTargetValue RequestMatchTarget diff --git a/internal/endpoint/handler/http/mock/protocol_options_test.go b/internal/endpoint/handler/http/mock/protocol_options_test.go index e56bc7e..abad022 100644 --- a/internal/endpoint/handler/http/mock/protocol_options_test.go +++ b/internal/endpoint/handler/http/mock/protocol_options_test.go @@ -95,6 +95,34 @@ rules: }, wantErr: false, }, + { + name: "Parse config with header matcher and TLS true", + args: args{ + config: ` +tls: true +rules: +- pattern: "^application/octet-stream$" + target: Content-Type + matcher: Header + response: ./assets/fakeFiles/sample.exe +`, + }, + wantOptions: httpOptions{ + TLS: true, + Rules: []targetRule{ + { + pattern: regexp.MustCompile("^application/octet-stream$"), + response: func() string { + p, _ := filepath.Abs("./assets/fakeFiles/sample.exe") + return p + }(), + requestMatchTarget: RequestMatchTargetHeader, + targetKey: "Content-Type", + }, + }, + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/audit/event_stream_test.go b/pkg/audit/event_stream_test.go index f89c981..e6b0e3d 100644 --- a/pkg/audit/event_stream_test.go +++ b/pkg/audit/event_stream_test.go @@ -27,8 +27,8 @@ var ( SourcePort: 32344, DestinationPort: 80, TLS: &audit.TLSDetails{ - Version: tls.VersionTLS13, - CipherSuite: tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + Version: audit.TLSVersionToEntity(tls.VersionTLS13).String(), + CipherSuite: tls.CipherSuiteName(tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA), ServerName: "localhost", }, ProtocolDetails: details.HTTP{ diff --git a/pkg/audit/sink/log_sink.go b/pkg/audit/sink/log_sink.go index fb3c131..52e9a5f 100644 --- a/pkg/audit/sink/log_sink.go +++ b/pkg/audit/sink/log_sink.go @@ -1,8 +1,6 @@ package sink import ( - "crypto/tls" - "gitlab.com/inetmock/inetmock/pkg/audit" "gitlab.com/inetmock/inetmock/pkg/logging" "go.uber.org/zap" @@ -34,7 +32,8 @@ func (l logSink) OnSubscribe(evs <-chan audit.Event, _ audit.CloseHandle) { if ev.TLS != nil { eventLogger = eventLogger.With( zap.String("tls_server_name", ev.TLS.ServerName), - zap.String("tls_cipher_suite", tls.CipherSuiteName(ev.TLS.CipherSuite)), + zap.String("tls_cipher_suite", ev.TLS.CipherSuite), + zap.String("tls_version", ev.TLS.Version), ) } diff --git a/pkg/audit/sink/writer_sink_test.go b/pkg/audit/sink/writer_sink_test.go index 0791369..cb955b9 100644 --- a/pkg/audit/sink/writer_sink_test.go +++ b/pkg/audit/sink/writer_sink_test.go @@ -27,8 +27,8 @@ var ( SourcePort: 32344, DestinationPort: 80, TLS: &audit.TLSDetails{ - Version: tls.VersionTLS13, - CipherSuite: tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + Version: audit.TLSVersionToEntity(tls.VersionTLS13).String(), + CipherSuite: tls.CipherSuiteName(tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA), ServerName: "localhost", }, ProtocolDetails: details.HTTP{ diff --git a/pkg/audit/tls_details.go b/pkg/audit/tls_details.go index a984007..e4565ff 100644 --- a/pkg/audit/tls_details.go +++ b/pkg/audit/tls_details.go @@ -1,8 +1,6 @@ package audit -import ( - "crypto/tls" -) +import "crypto/tls" var ( tlsToEntity = map[uint16]TLSVersion{ @@ -12,45 +10,37 @@ var ( tls.VersionTLS12: TLSVersion_TLS12, tls.VersionTLS13: TLSVersion_TLS13, } - entityToTls = map[TLSVersion]uint16{ - TLSVersion_SSLv30: tls.VersionSSL30, - TLSVersion_TLS10: tls.VersionTLS10, - TLSVersion_TLS11: tls.VersionTLS11, - TLSVersion_TLS12: tls.VersionTLS12, - TLSVersion_TLS13: tls.VersionTLS13, - } - cipherSuiteIDLookup = func(name string) uint16 { - for _, cs := range tls.CipherSuites() { - if cs.Name == name { - return cs.ID - } - } - return 0 - } ) type TLSDetails struct { - Version uint16 - CipherSuite uint16 + Version string + CipherSuite string ServerName string } +func TLSVersionToEntity(version uint16) TLSVersion { + if v, known := tlsToEntity[version]; known { + return v + } + return TLSVersion_SSLv30 +} + func NewTLSDetailsFromProto(entity *TLSDetailsEntity) *TLSDetails { if entity == nil { return nil } return &TLSDetails{ - Version: entityToTls[entity.GetVersion()], - CipherSuite: cipherSuiteIDLookup(entity.GetCipherSuite()), + Version: entity.GetVersion().String(), + CipherSuite: entity.GetCipherSuite(), ServerName: entity.GetServerName(), } } func (d TLSDetails) ProtoMessage() *TLSDetailsEntity { return &TLSDetailsEntity{ - Version: tlsToEntity[d.Version], - CipherSuite: tls.CipherSuiteName(d.CipherSuite), + Version: TLSVersion(TLSVersion_value[d.Version]), + CipherSuite: d.CipherSuite, ServerName: d.ServerName, } } From cc7259539899cea27aaaae746bbe4e00bf1c8589 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Tue, 26 Jan 2021 18:50:53 +0100 Subject: [PATCH 21/26] Subscribe sinks with a context to automatically remove them when context is canceled --- internal/app/app.go | 2 +- internal/rpc/audit_server.go | 4 ++-- pkg/audit/api.go | 7 +++---- pkg/audit/event_stream.go | 10 +++++++--- pkg/audit/event_stream_test.go | 9 +++++---- pkg/audit/sink/grpc_sink.go | 19 ++++++------------- pkg/audit/sink/log_sink.go | 2 +- pkg/audit/sink/log_sink_test.go | 3 ++- pkg/audit/sink/writer_sink.go | 2 +- pkg/audit/sink/writer_sink_test.go | 3 ++- 10 files changed, 30 insertions(+), 31 deletions(-) diff --git a/internal/app/app.go b/internal/app/app.go index 48c137b..871b737 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -286,7 +286,7 @@ func (a *app) WithEventStream() App { return } - if err = eventStream.RegisterSink(sink.NewLogSink(a.Logger().Named("LogSink"))); err != nil { + if err = eventStream.RegisterSink(a.ctx, sink.NewLogSink(a.Logger().Named("LogSink"))); err != nil { return } diff --git a/internal/rpc/audit_server.go b/internal/rpc/audit_server.go index 0fcfa8a..b9ac4da 100644 --- a/internal/rpc/audit_server.go +++ b/internal/rpc/audit_server.go @@ -25,7 +25,7 @@ func (a *auditServer) ListSinks(context.Context, *ListSinksRequest) (*ListSinksR func (a *auditServer) WatchEvents(req *WatchEventsRequest, srv Audit_WatchEventsServer) (err error) { a.logger.Info("watcher attached", zap.String("name", req.WatcherName)) - err = a.eventStream.RegisterSink(sink.NewGRPCSink(srv.Context(), req.WatcherName, func(ev audit.Event) { + err = a.eventStream.RegisterSink(srv.Context(), sink.NewGRPCSink(req.WatcherName, func(ev audit.Event) { if err = srv.Send(ev.ProtoMessage()); err != nil { return } @@ -59,7 +59,7 @@ func (a *auditServer) RegisterFileSink(_ context.Context, req *RegisterFileSinkR if writer, err = os.OpenFile(req.TargetPath, flags, permissions); err != nil { return } - if err = a.eventStream.RegisterSink(sink.NewWriterSink(req.TargetPath, audit.NewEventWriter(writer))); err != nil { + if err = a.eventStream.RegisterSink(context.Background(), sink.NewWriterSink(req.TargetPath, audit.NewEventWriter(writer))); err != nil { return } resp = &RegisterFileSinkResponse{} diff --git a/pkg/audit/api.go b/pkg/audit/api.go index 4129a2d..b3a789f 100644 --- a/pkg/audit/api.go +++ b/pkg/audit/api.go @@ -3,6 +3,7 @@ package audit import ( + "context" "errors" "io" ) @@ -15,17 +16,15 @@ type Emitter interface { Emit(ev Event) } -type CloseHandle func() - type Sink interface { Name() string - OnSubscribe(evs <-chan Event, close CloseHandle) + OnSubscribe(evs <-chan Event) } type EventStream interface { io.Closer Emitter - RegisterSink(s Sink) error + RegisterSink(ctx context.Context, s Sink) error Sinks() []string RemoveSink(name string) (exists bool) } diff --git a/pkg/audit/event_stream.go b/pkg/audit/event_stream.go index 6dbd522..262a29d 100644 --- a/pkg/audit/event_stream.go +++ b/pkg/audit/event_stream.go @@ -1,6 +1,7 @@ package audit import ( + "context" "sync" "time" @@ -89,7 +90,7 @@ func (e *eventStream) RemoveSink(name string) (exists bool) { return } -func (e *eventStream) RegisterSink(s Sink) error { +func (e *eventStream) RegisterSink(ctx context.Context, s Sink) error { name := s.Name() if _, exists := e.sinks[name]; exists { @@ -101,9 +102,12 @@ func (e *eventStream) RegisterSink(s Sink) error { lock: new(sync.Mutex), } - s.OnSubscribe(rs.downstream, func() { + s.OnSubscribe(rs.downstream) + + go func() { + <-ctx.Done() e.RemoveSink(name) - }) + }() e.sinks[name] = rs return nil diff --git a/pkg/audit/event_stream_test.go b/pkg/audit/event_stream_test.go index e6b0e3d..5be54a2 100644 --- a/pkg/audit/event_stream_test.go +++ b/pkg/audit/event_stream_test.go @@ -1,6 +1,7 @@ package audit_test import ( + "context" "crypto/tls" "net" "net/http" @@ -61,7 +62,7 @@ func (t *testSink) Name() string { return t.name } -func (t *testSink) OnSubscribe(evs <-chan audit.Event, _ audit.CloseHandle) { +func (t *testSink) OnSubscribe(evs <-chan audit.Event) { go func() { for ev := range evs { if t.consumer != nil { @@ -105,7 +106,7 @@ func Test_eventStream_RegisterSink(t *testing.T) { s: defaultSink, }, setup: func(e audit.EventStream) { - _ = e.RegisterSink(defaultSink) + _ = e.RegisterSink(context.Background(), defaultSink) }, wantErr: true, }, @@ -126,7 +127,7 @@ func Test_eventStream_RegisterSink(t *testing.T) { tt.setup(e) } - if err := e.RegisterSink(tt.args.s); (err != nil) != tt.wantErr { + if err := e.RegisterSink(context.Background(), tt.args.s); (err != nil) != tt.wantErr { t.Errorf("RegisterSink() error = %v, wantErr %v", err, tt.wantErr) } @@ -203,7 +204,7 @@ func Test_eventStream_Emit(t *testing.T) { if tt.subscribe { receivedWaitGroup.Add(len(tt.args.evs)) - if err := e.RegisterSink(wgMockSink(t, receivedWaitGroup)); err != nil { + if err := e.RegisterSink(context.Background(), wgMockSink(t, receivedWaitGroup)); err != nil { t.Errorf("RegisterSink() error = %v", err) } } diff --git a/pkg/audit/sink/grpc_sink.go b/pkg/audit/sink/grpc_sink.go index de52156..ee74446 100644 --- a/pkg/audit/sink/grpc_sink.go +++ b/pkg/audit/sink/grpc_sink.go @@ -6,10 +6,9 @@ import ( "gitlab.com/inetmock/inetmock/pkg/audit" ) -func NewGRPCSink(ctx context.Context, name string, consumer func(ev audit.Event)) audit.Sink { +func NewGRPCSink(name string, consumer func(ev audit.Event)) audit.Sink { return &grpcSink{ name: name, - ctx: ctx, consumer: consumer, } } @@ -24,16 +23,10 @@ func (g grpcSink) Name() string { return g.name } -func (g grpcSink) OnSubscribe(evs <-chan audit.Event, handle audit.CloseHandle) { - go func(ctx context.Context, consumer func(ev audit.Event), evs <-chan audit.Event, handle audit.CloseHandle) { - for { - select { - case ev := <-evs: - consumer(ev) - case <-ctx.Done(): - handle() - return - } +func (g grpcSink) OnSubscribe(evs <-chan audit.Event) { + go func(consumer func(ev audit.Event), evs <-chan audit.Event) { + for ev := range evs { + consumer(ev) } - }(g.ctx, g.consumer, evs, handle) + }(g.consumer, evs) } diff --git a/pkg/audit/sink/log_sink.go b/pkg/audit/sink/log_sink.go index 52e9a5f..efebc0b 100644 --- a/pkg/audit/sink/log_sink.go +++ b/pkg/audit/sink/log_sink.go @@ -24,7 +24,7 @@ func (logSink) Name() string { return logSinkName } -func (l logSink) OnSubscribe(evs <-chan audit.Event, _ audit.CloseHandle) { +func (l logSink) OnSubscribe(evs <-chan audit.Event) { go func(logger logging.Logger, evs <-chan audit.Event) { for ev := range evs { eventLogger := logger diff --git a/pkg/audit/sink/log_sink_test.go b/pkg/audit/sink/log_sink_test.go index c64f200..fe072d7 100644 --- a/pkg/audit/sink/log_sink_test.go +++ b/pkg/audit/sink/log_sink_test.go @@ -1,6 +1,7 @@ package sink_test import ( + "context" "sync" "testing" "time" @@ -90,7 +91,7 @@ func Test_logSink_OnSubscribe(t *testing.T) { t.Errorf("NewEventStream() error = %v", err) } - if err = evs.RegisterSink(logSink); err != nil { + if err = evs.RegisterSink(context.Background(), logSink); err != nil { t.Errorf("RegisterSink() error = %v", err) } diff --git a/pkg/audit/sink/writer_sink.go b/pkg/audit/sink/writer_sink.go index 5409dde..e0e2ba5 100644 --- a/pkg/audit/sink/writer_sink.go +++ b/pkg/audit/sink/writer_sink.go @@ -33,7 +33,7 @@ func (f writerCloserSink) Name() string { return f.name } -func (f writerCloserSink) OnSubscribe(evs <-chan audit.Event, _ audit.CloseHandle) { +func (f writerCloserSink) OnSubscribe(evs <-chan audit.Event) { go func(target audit.Writer, closeOnExit bool, evs <-chan audit.Event) { for ev := range evs { _ = target.Write(&ev) diff --git a/pkg/audit/sink/writer_sink_test.go b/pkg/audit/sink/writer_sink_test.go index cb955b9..885a506 100644 --- a/pkg/audit/sink/writer_sink_test.go +++ b/pkg/audit/sink/writer_sink_test.go @@ -1,6 +1,7 @@ package sink_test import ( + "context" "crypto/tls" "net" "net/http" @@ -91,7 +92,7 @@ func Test_writerCloserSink_OnSubscribe(t *testing.T) { t.Errorf("NewEventStream() error = %v", err) } - if err = evs.RegisterSink(writerCloserSink); err != nil { + if err = evs.RegisterSink(context.Background(), writerCloserSink); err != nil { t.Errorf("RegisterSink() error = %v", err) } From 2c02b0c9412272231d51702108f72b8b08f06138 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Tue, 26 Jan 2021 19:00:46 +0100 Subject: [PATCH 22/26] Add basic metric sink --- internal/app/app.go | 9 +++++++++ pkg/audit/sink/metric_sink.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 pkg/audit/sink/metric_sink.go diff --git a/internal/app/app.go b/internal/app/app.go index 871b737..aeb0494 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -290,6 +290,15 @@ func (a *app) WithEventStream() App { return } + var metricSink audit.Sink + if metricSink, err = sink.NewMetricSink(); err != nil { + return + } + + if err = eventStream.RegisterSink(a.ctx, metricSink); err != nil { + return + } + a.ctx = context.WithValue(a.ctx, eventStreamKey, eventStream) return }) diff --git a/pkg/audit/sink/metric_sink.go b/pkg/audit/sink/metric_sink.go new file mode 100644 index 0000000..2090dcb --- /dev/null +++ b/pkg/audit/sink/metric_sink.go @@ -0,0 +1,34 @@ +package sink + +import ( + "github.com/prometheus/client_golang/prometheus" + "gitlab.com/inetmock/inetmock/pkg/audit" + "gitlab.com/inetmock/inetmock/pkg/metrics" +) + +func NewMetricSink() (sink audit.Sink, err error) { + var totalEventsCounter *prometheus.CounterVec + if totalEventsCounter, err = metrics.Counter("audit", "events_total", "", "application", "transport"); err != nil { + return + } + sink = &metricSink{ + eventCounter: totalEventsCounter, + } + return +} + +type metricSink struct { + eventCounter *prometheus.CounterVec +} + +func (metricSink) Name() string { + return "metrics" +} + +func (m metricSink) OnSubscribe(evs <-chan audit.Event) { + go func(evs <-chan audit.Event) { + for ev := range evs { + m.eventCounter.WithLabelValues(ev.Application.String(), ev.Transport.String()).Inc() + } + }(evs) +} From c97207e85429c00d362eb61551ad1e95375459b2 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Wed, 27 Jan 2021 09:28:51 +0100 Subject: [PATCH 23/26] Update builtin CA --- assets/demoCA/ca.key | 5 +++++ assets/demoCA/ca.pem | 14 ++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 assets/demoCA/ca.key create mode 100644 assets/demoCA/ca.pem diff --git a/assets/demoCA/ca.key b/assets/demoCA/ca.key new file mode 100644 index 0000000..c77e1ea --- /dev/null +++ b/assets/demoCA/ca.key @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgEk8KFe6Vl3xefH82 +Na91uPQkBz4D/s1xE+59+kaRM1+hRANCAASnjLNiOc4UNsRBIQN+FOv8CEd6ftDH +Egg2dWUgGCFsgE2VaEG+jOwamBzTjCWbr0azIc+Brupd0CAChq8hVeaM +-----END PRIVATE KEY----- diff --git a/assets/demoCA/ca.pem b/assets/demoCA/ca.pem new file mode 100644 index 0000000..a815780 --- /dev/null +++ b/assets/demoCA/ca.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICGjCCAb+gAwIBAgIQZutydXTVGWPa32GhLwn4tDAKBggqhkjOPQQDAjBcMRAw +DgYDVQQGEwdnZXJtYW55MQwwCgYDVQQIEwNOUlcxETAPBgNVBAcTCERvcnRtdW5k +MREwDwYDVQQKEwhJTmV0TW9jazEUMBIGA1UEAxMLSU5ldE1vY2sgQ0EwIBcNMDEw +MjAxMDgyODA5WhgPMjA1MTAxMjAwODI4MDlaMFwxEDAOBgNVBAYTB2dlcm1hbnkx +DDAKBgNVBAgTA05SVzERMA8GA1UEBxMIRG9ydG11bmQxETAPBgNVBAoTCElOZXRN +b2NrMRQwEgYDVQQDEwtJTmV0TW9jayBDQTBZMBMGByqGSM49AgEGCCqGSM49AwEH +A0IABKeMs2I5zhQ2xEEhA34U6/wIR3p+0McSCDZ1ZSAYIWyATZVoQb6M7BqYHNOM +JZuvRrMhz4Gu6l3QIAKGryFV5oyjYTBfMA4GA1UdDwEB/wQEAwIChDAdBgNVHSUE +FjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQUkD5REm4Z34O8em3l5cXmHgd3S4swCgYIKoZIzj0EAwIDSQAwRgIhAJ5xWTz/ +/51Jyo7sO4hi3FhyDYJXeC0koRSQCvMggWuPAiEAjpgtX/UFukTAfTFsDp8q3AXZ +Kn0ejVWjmkh4K7nEJ5s= +-----END CERTIFICATE----- From 55dfcfabab875e8b99028d9f4582a7e6f104966d Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Wed, 27 Jan 2021 10:23:29 +0100 Subject: [PATCH 24/26] Ship default CA with container build --- .goreleaser.yml | 2 +- build/docker/inetmock.dockerfile | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 027d4f3..be5c7a6 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -81,4 +81,4 @@ dockers: - "--label=org.opencontainers.image.version={{.Version}}" extra_files: - config-container.yaml - - assets/fakeFiles/ + - assets/ diff --git a/build/docker/inetmock.dockerfile b/build/docker/inetmock.dockerfile index 33a2e3f..0fc09fe 100644 --- a/build/docker/inetmock.dockerfile +++ b/build/docker/inetmock.dockerfile @@ -1,5 +1,5 @@ # Runtime layer -FROM alpine:3.12 +FROM alpine:3.13 # Create appuser and group. ARG USER=inetmock @@ -20,6 +20,7 @@ RUN addgroup -S -g "${GROUP_ID}" "${GROUP}" && \ COPY --chown=$USER:$GROUP inetmock imctl /usr/lib/inetmock/bin/ COPY --chown=$USER:$GROUP assets/fakeFiles /var/lib/inetmock/fakeFiles/ +COPY --chown=$USER:$GROUP assets/demoCA /var/lib/inetmock/ca COPY config-container.yaml /etc/inetmock/config.yaml RUN mkdir -p /var/run/inetmock /var/lib/inetmock/certs /usr/lib/inetmock && \ From 85b012371124444a5fb44c782c4207d468b2c878 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Wed, 27 Jan 2021 11:29:10 +0100 Subject: [PATCH 25/26] Update container config --- config-container.yaml | 58 +++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/config-container.yaml b/config-container.yaml index f449288..636f536 100644 --- a/config-container.yaml +++ b/config-container.yaml @@ -1,18 +1,44 @@ x-response-rules: &httpResponseRules rules: - pattern: ".*\\.(?i)exe" + matcher: Path + - pattern: "^application/octet-stream$" + target: Accept + matcher: Header response: /var/lib/inetmock/fakeFiles/sample.exe - - pattern: ".*\\.(?i)(jpg|jpeg)" + - pattern: "^image/jpeg$" + target: Accept + matcher: Header response: /var/lib/inetmock/fakeFiles/default.jpg + - pattern: ".*\\.(?i)(jpg|jpeg)" + matcher: Path + response: /var/lib/inetmock/fakeFiles/default.jpg + - pattern: "^image/png$" + target: Accept + matcher: Header + response: /var/lib/inetmock/fakeFiles/default.png - pattern: ".*\\.(?i)png" + matcher: Path response: /var/lib/inetmock/fakeFiles/default.png - pattern: ".*\\.(?i)gif" + matcher: Path response: /var/lib/inetmock/fakeFiles/default.gif - pattern: ".*\\.(?i)ico" + matcher: Path response: /var/lib/inetmock/fakeFiles/default.ico - - pattern: ".*\\.(?i)txt" + - pattern: "^text/plain$" + target: Accept + matcher: Header response: /var/lib/inetmock/fakeFiles/default.txt + - pattern: ".*\\.(?i)txt" + matcher: Path + response: /var/lib/inetmock/fakeFiles/default.txt + - pattern: "^text/html$" + target: Accept + matcher: Header + response: /var/lib/inetmock/fakeFiles/default.html - pattern: ".*" + matcher: Path response: /var/lib/inetmock/fakeFiles/default.html api: @@ -43,6 +69,15 @@ endpoints: - 8080 options: <<: *httpResponseRules + https: + handler: http_mock + listenAddress: 0.0.0.0 + ports: + - 443 + - 8443 + options: + tls: true + <<: *httpResponseRules proxy: handler: http_proxy listenAddress: 0.0.0.0 @@ -52,16 +87,6 @@ endpoints: target: ipAddress: 127.0.0.1 port: 80 - httpsDowngrade: - handler: tls_interceptor - listenAddress: 0.0.0.0 - ports: - - 443 - - 8443 - options: - target: - ipAddress: 127.0.0.1 - port: 80 plainDns: handler: dns_mock listenAddress: 0.0.0.0 @@ -85,4 +110,11 @@ endpoints: options: target: ipAddress: 127.0.0.1 - port: 53 \ No newline at end of file + port: 53 + metrics: + handler: metrics_exporter + listenAddress: 0.0.0.0 + ports: + - 9110 + options: + route: /metrics \ No newline at end of file From 2d97beaf70baa88bbcabf89676314834208fd3eb Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Wed, 27 Jan 2021 13:31:51 +0100 Subject: [PATCH 26/26] Refactor gRPC sink - rename it to GenericSink - add tests - update dependencies - improve test step to always get a report - add NoOpSink for tests --- Taskfile.yml | 3 +- go.mod | 16 +-- go.sum | 204 ++++++++++++++++++++++++++++ internal/rpc/audit_server.go | 2 +- pkg/audit/event_stream_test.go | 38 ++---- pkg/audit/sink/generic_sink.go | 33 +++++ pkg/audit/sink/generic_sink_test.go | 65 +++++++++ pkg/audit/sink/grpc_sink.go | 32 ----- pkg/audit/sink/log_sink_test.go | 4 +- pkg/audit/sink/writer_sink_test.go | 8 +- 10 files changed, 331 insertions(+), 74 deletions(-) create mode 100644 pkg/audit/sink/generic_sink.go create mode 100644 pkg/audit/sink/generic_sink_test.go delete mode 100644 pkg/audit/sink/grpc_sink.go diff --git a/Taskfile.yml b/Taskfile.yml index e58d684..169d953 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -46,7 +46,8 @@ tasks: - generate cmds: - mkdir -p {{ .OUT_DIR }} - - go test -coverprofile={{ .OUT_DIR }}/cov-raw.out -covermode count -v ./... 2>&1 | tee {{ .OUT_DIR }}/test_output + - cmd: go test -coverprofile={{ .OUT_DIR }}/cov-raw.out -covermode count -v ./... 2>&1 | tee {{ .OUT_DIR }}/test_output + ignore_error: true - cat {{ .OUT_DIR }}/test_output | go-junit-report -set-exit-code > {{ .OUT_DIR }}/report.xml - grep -v "generated" {{ .OUT_DIR }}/cov-raw.out > {{ .OUT_DIR }}/cov.out - gocover-cobertura < {{ .OUT_DIR }}/cov.out > {{ .OUT_DIR }}/coverage.xml diff --git a/go.mod b/go.mod index aff09f2..bd4d26f 100644 --- a/go.mod +++ b/go.mod @@ -5,19 +5,19 @@ go 1.15 require ( github.com/bwmarrin/snowflake v0.3.0 github.com/golang/mock v1.4.4 - github.com/golang/protobuf v1.4.2 - github.com/google/uuid v1.1.2 + github.com/golang/protobuf v1.4.3 + github.com/google/uuid v1.2.0 github.com/imdario/mergo v0.3.11 - github.com/miekg/dns v1.1.31 + github.com/miekg/dns v1.1.35 github.com/olekukonko/tablewriter v0.0.4 - github.com/prometheus/client_golang v1.7.1 - github.com/spf13/cobra v1.0.0 + github.com/prometheus/client_golang v1.9.0 + github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 - go.uber.org/multierr v1.5.0 + go.uber.org/multierr v1.6.0 go.uber.org/zap v1.16.0 - google.golang.org/grpc v1.34.0 + google.golang.org/grpc v1.35.0 google.golang.org/protobuf v1.25.0 gopkg.in/elazarl/goproxy.v1 v1.0.0-20180725130230-947c36da3153 - gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/go.sum b/go.sum index d1a0e24..df50c82 100644 --- a/go.sum +++ b/go.sum @@ -14,15 +14,27 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -32,36 +44,55 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20201021153353-00ad82a08272 h1:Am81SElhR3XCQBunTisljzNkNese2T1FiV8jP79+dqg= github.com/elazarl/goproxy v0.0.0-20201021153353-00ad82a08272/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/elazarl/goproxy/ext v0.0.0-20201021153353-00ad82a08272 h1:xOHQWEGsftkjjFV2KIrC9vOz+iOinvZ7H6EAjSznqUk= github.com/elazarl/goproxy/ext v0.0.0-20201021153353-00ad82a08272/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= @@ -70,13 +101,20 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -93,6 +131,9 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -107,19 +148,30 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -130,6 +182,7 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -139,20 +192,29 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -160,6 +222,9 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -167,14 +232,20 @@ github.com/magiconair/properties v1.8.4 h1:8KGKTcQQGm0Kv7vEbKFErAoAOFyyacLStRtQS github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg= +github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs= +github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -186,65 +257,121 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.0 h1:7ks8ZkOP5/ujthUsT07rNv+nkLXCQWKNHuwzOAesEks= github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU= +github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= +github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.3.0 h1:Uehi/mxLK0eiUc0H0++5tpMGTexB8wZ598MIgU8VpDM= +github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -254,18 +381,26 @@ github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -276,19 +411,30 @@ github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= @@ -296,6 +442,7 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -304,9 +451,14 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -329,11 +481,13 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -344,8 +498,13 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -360,9 +519,11 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -372,13 +533,26 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201223074533-0d417f636930 h1:vRgIt+nup/B/BwIS0g2oC0haq0iqbV3ZA+u6+0TlNCo= golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -386,9 +560,14 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -411,16 +590,19 @@ golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktyp golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -429,6 +611,7 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -436,15 +619,24 @@ google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgX google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506 h1:uLBY0yHDCj2PMQ98KWDSIDFwn9zK2zh+tgWtbvPPBjI= +google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -454,6 +646,7 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -462,14 +655,19 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/elazarl/goproxy.v1 v1.0.0-20180725130230-947c36da3153 h1:i2sumy6EgvN2dbX7HPhoDc7hLyoym3OYdU5HlvUUrpE= gopkg.in/elazarl/goproxy.v1 v1.0.0-20180725130230-947c36da3153/go.mod h1:xzjpkyedLMz3EXUTBbkRuuGPsxfsBX3Sy7J6kC9Gvoc= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -477,12 +675,16 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -490,3 +692,5 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/internal/rpc/audit_server.go b/internal/rpc/audit_server.go index b9ac4da..a593179 100644 --- a/internal/rpc/audit_server.go +++ b/internal/rpc/audit_server.go @@ -25,7 +25,7 @@ func (a *auditServer) ListSinks(context.Context, *ListSinksRequest) (*ListSinksR func (a *auditServer) WatchEvents(req *WatchEventsRequest, srv Audit_WatchEventsServer) (err error) { a.logger.Info("watcher attached", zap.String("name", req.WatcherName)) - err = a.eventStream.RegisterSink(srv.Context(), sink.NewGRPCSink(req.WatcherName, func(ev audit.Event) { + err = a.eventStream.RegisterSink(srv.Context(), sink.NewGenericSink(req.WatcherName, func(ev audit.Event) { if err = srv.Send(ev.ProtoMessage()); err != nil { return } diff --git a/pkg/audit/event_stream_test.go b/pkg/audit/event_stream_test.go index 5be54a2..bd15ea8 100644 --- a/pkg/audit/event_stream_test.go +++ b/pkg/audit/event_stream_test.go @@ -11,14 +11,13 @@ import ( "gitlab.com/inetmock/inetmock/pkg/audit" "gitlab.com/inetmock/inetmock/pkg/audit/details" + "gitlab.com/inetmock/inetmock/pkg/audit/sink" "gitlab.com/inetmock/inetmock/pkg/logging" "gitlab.com/inetmock/inetmock/pkg/wait" ) var ( - defaultSink = &testSink{ - name: "test defaultSink", - } + noOpSink = sink.NewNoOpSink("test defaultSink") testEvents = []*audit.Event{ { Transport: audit.TransportProtocol_TCP, @@ -53,33 +52,14 @@ var ( } ) -type testSink struct { - name string - consumer func(event audit.Event) -} - -func (t *testSink) Name() string { - return t.name -} - -func (t *testSink) OnSubscribe(evs <-chan audit.Event) { - go func() { - for ev := range evs { - if t.consumer != nil { - t.consumer(ev) - } - } - }() -} - func wgMockSink(t testing.TB, wg *sync.WaitGroup) audit.Sink { - return &testSink{ - name: "WG mock sink", - consumer: func(event audit.Event) { + return sink.NewGenericSink( + "WG mock sink", + func(event audit.Event) { t.Logf("Got event = %v", event) wg.Done() }, - } + ) } func Test_eventStream_RegisterSink(t *testing.T) { @@ -96,17 +76,17 @@ func Test_eventStream_RegisterSink(t *testing.T) { { name: "Register test defaultSink", args: args{ - s: defaultSink, + s: noOpSink, }, wantErr: false, }, { name: "Fail due to already registered defaultSink", args: args{ - s: defaultSink, + s: noOpSink, }, setup: func(e audit.EventStream) { - _ = e.RegisterSink(context.Background(), defaultSink) + _ = e.RegisterSink(context.Background(), noOpSink) }, wantErr: true, }, diff --git a/pkg/audit/sink/generic_sink.go b/pkg/audit/sink/generic_sink.go new file mode 100644 index 0000000..4c10e84 --- /dev/null +++ b/pkg/audit/sink/generic_sink.go @@ -0,0 +1,33 @@ +package sink + +import ( + "gitlab.com/inetmock/inetmock/pkg/audit" +) + +func NewNoOpSink(name string) audit.Sink { + return NewGenericSink(name, func(_ audit.Event) {}) +} + +func NewGenericSink(name string, consumer func(ev audit.Event)) audit.Sink { + return &genericSink{ + name: name, + consumer: consumer, + } +} + +type genericSink struct { + name string + consumer func(ev audit.Event) +} + +func (g genericSink) Name() string { + return g.name +} + +func (g genericSink) OnSubscribe(evs <-chan audit.Event) { + go func(consumer func(ev audit.Event), evs <-chan audit.Event) { + for ev := range evs { + consumer(ev) + } + }(g.consumer, evs) +} diff --git a/pkg/audit/sink/generic_sink_test.go b/pkg/audit/sink/generic_sink_test.go new file mode 100644 index 0000000..122f105 --- /dev/null +++ b/pkg/audit/sink/generic_sink_test.go @@ -0,0 +1,65 @@ +package sink_test + +import ( + "context" + "sync" + "testing" + "time" + + "gitlab.com/inetmock/inetmock/pkg/audit" + "gitlab.com/inetmock/inetmock/pkg/audit/sink" + "gitlab.com/inetmock/inetmock/pkg/logging" + "gitlab.com/inetmock/inetmock/pkg/wait" +) + +func Test_genericSink_OnSubscribe(t *testing.T) { + type testCase struct { + name string + events []*audit.Event + } + tests := []testCase{ + { + name: "Get a single log line", + events: testEvents[:1], + }, + { + name: "Get multiple events", + events: testEvents, + }, + } + scenario := func(tt testCase) func(t *testing.T) { + return func(t *testing.T) { + wg := new(sync.WaitGroup) + wg.Add(len(tt.events)) + + genericSink := sink.NewGenericSink(t.Name(), func(ev audit.Event) { + wg.Done() + }) + + var evs audit.EventStream + var err error + if evs, err = audit.NewEventStream(logging.CreateTestLogger(t)); err != nil { + t.Errorf("NewEventStream() error = %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + if err = evs.RegisterSink(ctx, genericSink); err != nil { + t.Errorf("RegisterSink() error = %v", err) + } + + for _, ev := range tt.events { + evs.Emit(*ev) + } + + select { + case <-time.After(100 * time.Millisecond): + t.Errorf("not all events recorded in time") + case <-wait.ForWaitGroupDone(wg): + } + } + } + for _, tt := range tests { + t.Run(tt.name, scenario(tt)) + } +} diff --git a/pkg/audit/sink/grpc_sink.go b/pkg/audit/sink/grpc_sink.go deleted file mode 100644 index ee74446..0000000 --- a/pkg/audit/sink/grpc_sink.go +++ /dev/null @@ -1,32 +0,0 @@ -package sink - -import ( - "context" - - "gitlab.com/inetmock/inetmock/pkg/audit" -) - -func NewGRPCSink(name string, consumer func(ev audit.Event)) audit.Sink { - return &grpcSink{ - name: name, - consumer: consumer, - } -} - -type grpcSink struct { - name string - ctx context.Context - consumer func(ev audit.Event) -} - -func (g grpcSink) Name() string { - return g.name -} - -func (g grpcSink) OnSubscribe(evs <-chan audit.Event) { - go func(consumer func(ev audit.Event), evs <-chan audit.Event) { - for ev := range evs { - consumer(ev) - } - }(g.consumer, evs) -} diff --git a/pkg/audit/sink/log_sink_test.go b/pkg/audit/sink/log_sink_test.go index fe072d7..cc8deca 100644 --- a/pkg/audit/sink/log_sink_test.go +++ b/pkg/audit/sink/log_sink_test.go @@ -91,7 +91,9 @@ func Test_logSink_OnSubscribe(t *testing.T) { t.Errorf("NewEventStream() error = %v", err) } - if err = evs.RegisterSink(context.Background(), logSink); err != nil { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + if err = evs.RegisterSink(ctx, logSink); err != nil { t.Errorf("RegisterSink() error = %v", err) } diff --git a/pkg/audit/sink/writer_sink_test.go b/pkg/audit/sink/writer_sink_test.go index 885a506..6d59c32 100644 --- a/pkg/audit/sink/writer_sink_test.go +++ b/pkg/audit/sink/writer_sink_test.go @@ -75,6 +75,7 @@ func Test_writerCloserSink_OnSubscribe(t *testing.T) { ctrl := gomock.NewController(t) t.Cleanup(ctrl.Finish) + writerMock := audit_mock.NewMockWriter(ctrl) writerMock. EXPECT(). @@ -84,7 +85,7 @@ func Test_writerCloserSink_OnSubscribe(t *testing.T) { }). Times(len(tt.events)) - writerCloserSink := sink.NewWriterSink("WriterMock", writerMock, sink.WithCloseOnExit) + writerCloserSink := sink.NewWriterSink("WriterMock", writerMock) var evs audit.EventStream var err error @@ -92,7 +93,10 @@ func Test_writerCloserSink_OnSubscribe(t *testing.T) { t.Errorf("NewEventStream() error = %v", err) } - if err = evs.RegisterSink(context.Background(), writerCloserSink); err != nil { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + if err = evs.RegisterSink(ctx, writerCloserSink); err != nil { t.Errorf("RegisterSink() error = %v", err) }