config: add customization options for logging (#4383)

* config: add customization options for logging

* config: validate log fields

* allocate slices once
This commit is contained in:
Caleb Doxsey 2023-07-24 13:17:03 -06:00 committed by GitHub
parent 577319d26c
commit 438aecd7bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 946 additions and 548 deletions

View file

@ -28,23 +28,11 @@ func (a *Authorize) logAuthorizeCheck(
defer span.End()
hdrs := getCheckRequestHeaders(in)
hattrs := in.GetAttributes().GetRequest().GetHttp()
evt := log.Info(ctx).Str("service", "authorize")
// request
evt = evt.Str("request-id", requestid.FromContext(ctx))
evt = evt.Str("check-request-id", hdrs["X-Request-Id"])
evt = evt.Str("method", hattrs.GetMethod())
evt = evt.Str("path", stripQueryString(hattrs.GetPath()))
evt = evt.Str("host", hattrs.GetHost())
evt = evt.Str("query", hattrs.GetQuery())
evt = evt.Str("ip", in.GetAttributes().GetSource().GetAddress().GetSocketAddress().GetAddress())
impersonateDetails := a.getImpersonateDetails(ctx, s)
// session information
if s, ok := s.(*session.Session); ok {
evt = a.populateLogSessionDetails(ctx, evt, s)
}
if sa, ok := s.(*user.ServiceAccount); ok {
evt = evt.Str("service-account-id", sa.GetId())
evt := log.Info(ctx).Str("service", "authorize")
for _, field := range a.currentOptions.Load().GetAuthorizeLogFields() {
evt = populateLogEvent(ctx, field, evt, in, s, u, hdrs, impersonateDetails)
}
// result
@ -61,13 +49,6 @@ func (a *Authorize) logAuthorizeCheck(
} else {
evt = evt.Strs("deny-why-false", res.Deny.Reasons.Strings())
}
evt = evt.Str("user", u.GetId())
evt = evt.Str("email", u.GetEmail())
}
// potentially sensitive, only log if debug mode
if zerolog.GlobalLevel() <= zerolog.DebugLevel {
evt = evt.Interface("headers", hdrs)
}
evt.Msg("authorize check")
@ -92,56 +73,132 @@ func (a *Authorize) logAuthorizeCheck(
}
}
func (a *Authorize) populateLogSessionDetails(ctx context.Context, evt *zerolog.Event, s *session.Session) *zerolog.Event {
evt = evt.Str("session-id", s.GetId())
if s.GetImpersonateSessionId() == "" {
return evt
type impersonateDetails struct {
email string
sessionID string
userID string
}
func (a *Authorize) getImpersonateDetails(
ctx context.Context,
s sessionOrServiceAccount,
) *impersonateDetails {
var sessionID string
if s, ok := s.(*session.Session); ok {
sessionID = s.GetImpersonateSessionId()
}
if sessionID == "" {
return nil
}
querier := storage.GetQuerier(ctx)
evt = evt.Str("impersonate-session-id", s.GetImpersonateSessionId())
req := &databroker.QueryRequest{
Type: grpcutil.GetTypeURL(new(session.Session)),
Limit: 1,
}
req.SetFilterByID(s.GetImpersonateSessionId())
req.SetFilterByID(sessionID)
res, err := querier.Query(ctx, req)
if err != nil || len(res.GetRecords()) == 0 {
return evt
return nil
}
impersonatedSessionMsg, err := res.GetRecords()[0].GetData().UnmarshalNew()
if err != nil {
return evt
return nil
}
impersonatedSession, ok := impersonatedSessionMsg.(*session.Session)
if !ok {
return evt
return nil
}
evt = evt.Str("impersonate-user-id", impersonatedSession.GetUserId())
userID := impersonatedSession.GetUserId()
req = &databroker.QueryRequest{
Type: grpcutil.GetTypeURL(new(user.User)),
Limit: 1,
}
req.SetFilterByID(impersonatedSession.GetUserId())
req.SetFilterByID(userID)
res, err = querier.Query(ctx, req)
if err != nil || len(res.GetRecords()) == 0 {
return evt
return nil
}
impersonatedUserMsg, err := res.GetRecords()[0].GetData().UnmarshalNew()
if err != nil {
return evt
return nil
}
impersonatedUser, ok := impersonatedUserMsg.(*user.User)
if !ok {
return nil
}
email := impersonatedUser.GetEmail()
return &impersonateDetails{
sessionID: sessionID,
userID: userID,
email: email,
}
}
func populateLogEvent(
ctx context.Context,
field log.AuthorizeLogField,
evt *zerolog.Event,
in *envoy_service_auth_v3.CheckRequest,
s sessionOrServiceAccount,
u *user.User,
hdrs map[string]string,
impersonateDetails *impersonateDetails,
) *zerolog.Event {
switch field {
case log.AuthorizeLogFieldCheckRequestID:
return evt.Str(string(field), hdrs["X-Request-Id"])
case log.AuthorizeLogFieldEmail:
return evt.Str(string(field), u.GetEmail())
case log.AuthorizeLogFieldHeaders:
return evt.Interface(string(field), hdrs)
case log.AuthorizeLogFieldHost:
return evt.Str(string(field), in.GetAttributes().GetRequest().GetHttp().GetHost())
case log.AuthorizeLogFieldImpersonateEmail:
if impersonateDetails != nil {
evt = evt.Str(string(field), impersonateDetails.email)
}
return evt
case log.AuthorizeLogFieldImpersonateSessionID:
if impersonateDetails != nil {
evt = evt.Str(string(field), impersonateDetails.sessionID)
}
return evt
case log.AuthorizeLogFieldImpersonateUserID:
if impersonateDetails != nil {
evt = evt.Str(string(field), impersonateDetails.userID)
}
return evt
case log.AuthorizeLogFieldIP:
return evt.Str(string(field), in.GetAttributes().GetSource().GetAddress().GetSocketAddress().GetAddress())
case log.AuthorizeLogFieldMethod:
return evt.Str(string(field), in.GetAttributes().GetRequest().GetHttp().GetMethod())
case log.AuthorizeLogFieldPath:
return evt.Str(string(field), stripQueryString(in.GetAttributes().GetRequest().GetHttp().GetPath()))
case log.AuthorizeLogFieldQuery:
return evt.Str(string(field), in.GetAttributes().GetRequest().GetHttp().GetQuery())
case log.AuthorizeLogFieldRequestID:
return evt.Str(string(field), requestid.FromContext(ctx))
case log.AuthorizeLogFieldServiceAccountID:
if sa, ok := s.(*user.ServiceAccount); ok {
evt = evt.Str(string(field), sa.GetId())
}
return evt
case log.AuthorizeLogFieldSessionID:
if s, ok := s.(*session.Session); ok {
evt = evt.Str(string(field), s.GetId())
}
return evt
case log.AuthorizeLogFieldUser:
return evt.Str(string(field), u.GetId())
default:
return evt
}
evt = evt.Str("impersonate-email", impersonatedUser.GetEmail())
return evt
}
func stripQueryString(str string) string {

View file

@ -68,6 +68,12 @@ type Options struct {
// Possible options are "info","warn", and "error". Defaults to the value of `LogLevel`.
ProxyLogLevel LogLevel `mapstructure:"proxy_log_level" yaml:"proxy_log_level,omitempty"`
// AccessLogFields are the fields to log in access logs.
AccessLogFields []log.AccessLogField `mapstructure:"access_log_fields" yaml:"access_log_fields,omitempty"`
// AuthorizeLogFields are the fields to log in authorize logs.
AuthorizeLogFields []log.AuthorizeLogField `mapstructure:"authorize_log_fields" yaml:"authorize_log_fields,omitempty"`
// SharedKey is the shared secret authorization key used to mutually authenticate
// requests between services.
SharedKey string `mapstructure:"shared_secret" yaml:"shared_secret,omitempty"`
@ -749,6 +755,18 @@ func (o *Options) Validate() error {
return fmt.Errorf("config: invalid proxy_log_level: %w", err)
}
for _, field := range o.AccessLogFields {
if err := field.Validate(); err != nil {
return fmt.Errorf("config: invalid access_log_fields: %w", err)
}
}
for _, field := range o.AuthorizeLogFields {
if err := field.Validate(); err != nil {
return fmt.Errorf("config: invalid authorize_log_fields: %w", err)
}
}
return nil
}
@ -1283,6 +1301,22 @@ func (o *Options) GetSigningKey() ([]byte, error) {
return []byte(rawSigningKey), nil
}
// GetAccessLogFields returns the access log fields. If none are set, the default fields are returned.
func (o *Options) GetAccessLogFields() []log.AccessLogField {
if o.AccessLogFields == nil {
return log.DefaultAccessLogFields()
}
return o.AccessLogFields
}
// GetAuthorizeLogFields returns the authorize log fields. If none are set, the default fields are returned.
func (o *Options) GetAuthorizeLogFields() []log.AuthorizeLogField {
if o.AuthorizeLogFields == nil {
return log.DefaultAuthorizeLogFields()
}
return o.AuthorizeLogFields
}
// NewCookie creates a new Cookie.
func (o *Options) NewCookie() *http.Cookie {
return &http.Cookie{
@ -1329,6 +1363,8 @@ func (o *Options) ApplySettings(ctx context.Context, certsIndex *cryptutil.Certi
set(&o.InstallationID, settings.InstallationId)
set(&o.Debug, settings.Debug)
setLogLevel(&o.LogLevel, settings.LogLevel)
setAccessLogFields(&o.AccessLogFields, settings.AccessLogFields)
setAuthorizeLogFields(&o.AuthorizeLogFields, settings.AuthorizeLogFields)
setLogLevel(&o.ProxyLogLevel, settings.ProxyLogLevel)
set(&o.SharedKey, settings.SharedSecret)
set(&o.Services, settings.Services)
@ -1458,6 +1494,26 @@ func set[T any](dst, src *T) {
*dst = *src
}
func setAccessLogFields(dst *[]log.AccessLogField, src *config.Settings_StringList) {
if src == nil {
return
}
*dst = make([]log.AccessLogField, len(src.Values))
for i, v := range src.Values {
(*dst)[i] = log.AccessLogField(v)
}
}
func setAuthorizeLogFields(dst *[]log.AuthorizeLogField, src *config.Settings_StringList) {
if src == nil {
return
}
*dst = make([]log.AuthorizeLogField, len(src.Values))
for i, v := range src.Values {
(*dst)[i] = log.AuthorizeLogField(v)
}
}
func setAuditKey(dst **PublicKeyEncryptionKeyOptions, src *crypt.PublicKeyEncryptionKey) {
if src == nil {
return

View file

@ -3,6 +3,7 @@ package controlplane
import (
"strings"
envoy_data_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/data/accesslog/v3"
envoy_service_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v3"
"github.com/rs/zerolog"
@ -30,28 +31,51 @@ func (srv *Server) StreamAccessLogs(stream envoy_service_accesslog_v3.AccessLogS
} else {
evt = log.Info(stream.Context())
}
// common properties
evt = evt.Str("service", "envoy")
evt = evt.Str("upstream-cluster", entry.GetCommonProperties().GetUpstreamCluster())
// request properties
evt = evt.Str("method", entry.GetRequest().GetRequestMethod().String())
evt = evt.Str("authority", entry.GetRequest().GetAuthority())
evt = evt.Str("path", stripQueryString(reqPath))
evt = evt.Str("user-agent", entry.GetRequest().GetUserAgent())
evt = evt.Str("referer", stripQueryString(entry.GetRequest().GetReferer()))
evt = evt.Str("forwarded-for", entry.GetRequest().GetForwardedFor())
evt = evt.Str("request-id", entry.GetRequest().GetRequestId())
// response properties
dur := entry.CommonProperties.TimeToLastDownstreamTxByte.AsDuration()
evt = evt.Dur("duration", dur)
evt = evt.Uint64("size", entry.Response.ResponseBodyBytes)
evt = evt.Uint32("response-code", entry.GetResponse().GetResponseCode().GetValue())
evt = evt.Str("response-code-details", entry.GetResponse().GetResponseCodeDetails())
for _, field := range srv.currentConfig.Load().Config.Options.GetAccessLogFields() {
evt = populateLogEvent(field, evt, entry)
}
evt.Msg("http-request")
}
}
}
func populateLogEvent(
field log.AccessLogField,
evt *zerolog.Event,
entry *envoy_data_accesslog_v3.HTTPAccessLogEntry,
) *zerolog.Event {
switch field {
case log.AccessLogFieldAuthority:
return evt.Str(string(field), entry.GetRequest().GetAuthority())
case log.AccessLogFieldDuration:
dur := entry.CommonProperties.TimeToLastDownstreamTxByte.AsDuration()
return evt.Dur(string(field), dur)
case log.AccessLogFieldForwardedFor:
return evt.Str(string(field), entry.GetRequest().GetForwardedFor())
case log.AccessLogFieldMethod:
return evt.Str(string(field), entry.GetRequest().GetRequestMethod().String())
case log.AccessLogFieldPath:
return evt.Str(string(field), stripQueryString(entry.GetRequest().GetPath()))
case log.AccessLogFieldReferer:
return evt.Str(string(field), stripQueryString(entry.GetRequest().GetReferer()))
case log.AccessLogFieldRequestID:
return evt.Str(string(field), entry.GetRequest().GetRequestId())
case log.AccessLogFieldResponseCode:
return evt.Uint32(string(field), entry.GetResponse().GetResponseCode().GetValue())
case log.AccessLogFieldResponseCodeDetails:
return evt.Str(string(field), entry.GetResponse().GetResponseCodeDetails())
case log.AccessLogFieldSize:
return evt.Uint64(string(field), entry.Response.ResponseBodyBytes)
case log.AccessLogFieldUpstreamCluster:
return evt.Str(string(field), entry.GetCommonProperties().GetUpstreamCluster())
case log.AccessLogFieldUserAgent:
return evt.Str(string(field), entry.GetRequest().GetUserAgent())
default:
return evt
}
}
func stripQueryString(str string) string {
if idx := strings.Index(str, "?"); idx != -1 {
str = str[:idx]

73
internal/log/access.go Normal file
View file

@ -0,0 +1,73 @@
package log
import (
"errors"
"fmt"
)
// An AccessLogField is a field in the access logs.
type AccessLogField string
// known access log fields
const (
AccessLogFieldAuthority AccessLogField = "authority"
AccessLogFieldDuration AccessLogField = "duration"
AccessLogFieldForwardedFor AccessLogField = "forwarded-for"
AccessLogFieldMethod AccessLogField = "method"
AccessLogFieldPath AccessLogField = "path"
AccessLogFieldReferer AccessLogField = "referer"
AccessLogFieldRequestID AccessLogField = "request-id"
AccessLogFieldResponseCode AccessLogField = "response-code"
AccessLogFieldResponseCodeDetails AccessLogField = "response-code-details"
AccessLogFieldSize AccessLogField = "size"
AccessLogFieldUpstreamCluster AccessLogField = "upstream-cluster"
AccessLogFieldUserAgent AccessLogField = "user-agent"
)
var defaultAccessLogFields = []AccessLogField{
AccessLogFieldUpstreamCluster,
AccessLogFieldMethod,
AccessLogFieldAuthority,
AccessLogFieldPath,
AccessLogFieldUserAgent,
AccessLogFieldReferer,
AccessLogFieldForwardedFor,
AccessLogFieldRequestID,
AccessLogFieldDuration,
AccessLogFieldSize,
AccessLogFieldResponseCode,
AccessLogFieldResponseCodeDetails,
}
// DefaultAccessLogFields returns the default access log fields.
func DefaultAccessLogFields() []AccessLogField {
return defaultAccessLogFields
}
// ErrUnknownAccessLogField indicates that an access log field is unknown.
var ErrUnknownAccessLogField = errors.New("unknown access log field")
var accessLogFieldLookup = map[AccessLogField]struct{}{
AccessLogFieldAuthority: {},
AccessLogFieldDuration: {},
AccessLogFieldForwardedFor: {},
AccessLogFieldMethod: {},
AccessLogFieldPath: {},
AccessLogFieldReferer: {},
AccessLogFieldRequestID: {},
AccessLogFieldResponseCode: {},
AccessLogFieldResponseCodeDetails: {},
AccessLogFieldSize: {},
AccessLogFieldUpstreamCluster: {},
AccessLogFieldUserAgent: {},
}
// Validate returns an error if the access log field is invalid.
func (field AccessLogField) Validate() error {
_, ok := accessLogFieldLookup[field]
if !ok {
return fmt.Errorf("%w: %s", ErrUnknownAccessLogField, field)
}
return nil
}

88
internal/log/authorize.go Normal file
View file

@ -0,0 +1,88 @@
package log
import (
"errors"
"fmt"
"github.com/rs/zerolog"
)
// An AuthorizeLogField is a field in the authorize logs.
type AuthorizeLogField string
// known authorize log fields
const (
AuthorizeLogFieldCheckRequestID AuthorizeLogField = "check-request-id"
AuthorizeLogFieldEmail AuthorizeLogField = "email"
AuthorizeLogFieldHeaders AuthorizeLogField = "headers"
AuthorizeLogFieldHost AuthorizeLogField = "host"
AuthorizeLogFieldImpersonateEmail AuthorizeLogField = "impersonate-email"
AuthorizeLogFieldImpersonateSessionID AuthorizeLogField = "impersonate-session-id"
AuthorizeLogFieldImpersonateUserID AuthorizeLogField = "impersonate-user-id"
AuthorizeLogFieldIP AuthorizeLogField = "ip"
AuthorizeLogFieldMethod AuthorizeLogField = "method"
AuthorizeLogFieldPath AuthorizeLogField = "path"
AuthorizeLogFieldQuery AuthorizeLogField = "query"
AuthorizeLogFieldRequestID AuthorizeLogField = "request-id"
AuthorizeLogFieldServiceAccountID AuthorizeLogField = "service-account-id"
AuthorizeLogFieldSessionID AuthorizeLogField = "session-id"
AuthorizeLogFieldUser AuthorizeLogField = "user"
)
var defaultAuthorizeLogFields = []AuthorizeLogField{
AuthorizeLogFieldRequestID,
AuthorizeLogFieldCheckRequestID,
AuthorizeLogFieldMethod,
AuthorizeLogFieldPath,
AuthorizeLogFieldHost,
AuthorizeLogFieldQuery,
AuthorizeLogFieldIP,
AuthorizeLogFieldSessionID,
AuthorizeLogFieldImpersonateSessionID,
AuthorizeLogFieldImpersonateUserID,
AuthorizeLogFieldImpersonateEmail,
AuthorizeLogFieldServiceAccountID,
AuthorizeLogFieldUser,
AuthorizeLogFieldEmail,
}
var defaultDebugAuthorizeLogFields = append(defaultAuthorizeLogFields, AuthorizeLogFieldHeaders)
// DefaultAuthorizeLogFields returns the default authorize log fields.
func DefaultAuthorizeLogFields() []AuthorizeLogField {
if zerolog.GlobalLevel() <= zerolog.DebugLevel {
return defaultDebugAuthorizeLogFields
}
return defaultAuthorizeLogFields
}
// ErrUnknownAuthorizeLogField indicates that an authorize log field is unknown.
var ErrUnknownAuthorizeLogField = errors.New("unknown authorize log field")
var authorizeLogFieldLookup = map[AuthorizeLogField]struct{}{
AuthorizeLogFieldCheckRequestID: {},
AuthorizeLogFieldEmail: {},
AuthorizeLogFieldHeaders: {},
AuthorizeLogFieldHost: {},
AuthorizeLogFieldImpersonateEmail: {},
AuthorizeLogFieldImpersonateSessionID: {},
AuthorizeLogFieldImpersonateUserID: {},
AuthorizeLogFieldIP: {},
AuthorizeLogFieldMethod: {},
AuthorizeLogFieldPath: {},
AuthorizeLogFieldQuery: {},
AuthorizeLogFieldRequestID: {},
AuthorizeLogFieldServiceAccountID: {},
AuthorizeLogFieldSessionID: {},
AuthorizeLogFieldUser: {},
}
// Validate returns an error if the authorize log field is invalid.
func (field AuthorizeLogField) Validate() error {
_, ok := authorizeLogFieldLookup[field]
if !ok {
return fmt.Errorf("%w: %s", ErrUnknownAuthorizeLogField, field)
}
return nil
}

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc-gen-go v1.31.0
// protoc v3.21.7
// source: audit.proto

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc-gen-go v1.31.0
// protoc v3.21.7
// source: api.proto

File diff suppressed because it is too large Load diff

View file

@ -137,10 +137,15 @@ message Settings {
bytes cert_bytes = 3;
bytes key_bytes = 4;
}
message StringList {
repeated string values = 1;
}
optional string installation_id = 71;
optional bool debug = 2;
optional string log_level = 3;
optional StringList access_log_fields = 114;
optional StringList authorize_log_fields = 115;
optional string proxy_log_level = 4;
optional string shared_secret = 5;
optional string services = 6;

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc-gen-go v1.31.0
// protoc v3.21.7
// source: crypt.proto

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc-gen-go v1.31.0
// protoc v3.21.7
// source: databroker.proto

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc-gen-go v1.31.0
// protoc v3.21.7
// source: device.proto

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc-gen-go v1.31.0
// protoc v3.21.7
// source: last_error.proto

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc-gen-go v1.31.0
// protoc v3.21.7
// source: identity.proto

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc-gen-go v1.31.0
// protoc v3.21.7
// source: registry.proto

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc-gen-go v1.31.0
// protoc v3.21.7
// source: session.proto

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc-gen-go v1.31.0
// protoc v3.21.7
// source: user.proto