feature/databroker: user data and session refactor project (#926)

* databroker: add databroker, identity manager, update cache (#864)

* databroker: add databroker, identity manager, update cache

* fix cache tests

* directory service (#885)

* directory: add google and okta

* add onelogin

* add directory provider

* initialize before sync, upate google provider, remove dead code

* add azure provider

* fix azure provider

* fix gitlab

* add gitlab test, fix azure test

* hook up okta

* remove dead code

* fix tests

* fix flaky test

* authorize: use databroker data for rego policy (#904)

* wip

* add directory provider

* initialize before sync, upate google provider, remove dead code

* fix flaky test

* update authorize to use databroker data

* implement signed jwt

* wait for session and user to appear

* fix test

* directory service (#885)

* directory: add google and okta

* add onelogin

* add directory provider

* initialize before sync, upate google provider, remove dead code

* add azure provider

* fix azure provider

* fix gitlab

* add gitlab test, fix azure test

* hook up okta

* remove dead code

* fix tests

* fix flaky test

* remove log line

* only redirect when no session id exists

* prepare rego query as part of create

* return on ctx done

* retry on disconnect for sync

* move jwt signing

* use !=

* use parent ctx for wait

* remove session state, remove logs

* rename function

* add log message

* pre-allocate slice

* use errgroup

* return nil on eof for sync

* move check

* disable timeout on gRPC requests in envoy

* fix gitlab test

* use v4 backoff

* authenticate: databroker changes (#914)

* wip

* add directory provider

* initialize before sync, upate google provider, remove dead code

* fix flaky test

* update authorize to use databroker data

* implement signed jwt

* wait for session and user to appear

* fix test

* directory service (#885)

* directory: add google and okta

* add onelogin

* add directory provider

* initialize before sync, upate google provider, remove dead code

* add azure provider

* fix azure provider

* fix gitlab

* add gitlab test, fix azure test

* hook up okta

* remove dead code

* fix tests

* fix flaky test

* remove log line

* only redirect when no session id exists

* prepare rego query as part of create

* return on ctx done

* retry on disconnect for sync

* move jwt signing

* use !=

* use parent ctx for wait

* remove session state, remove logs

* rename function

* add log message

* pre-allocate slice

* use errgroup

* return nil on eof for sync

* move check

* disable timeout on gRPC requests in envoy

* fix dashboard

* delete session on logout

* permanently delete sessions once they are marked as deleted

* remove permanent delete

* fix tests

* remove groups and refresh test

* databroker: remove dead code, rename cache url, move dashboard (#925)

* wip

* add directory provider

* initialize before sync, upate google provider, remove dead code

* fix flaky test

* update authorize to use databroker data

* implement signed jwt

* wait for session and user to appear

* fix test

* directory service (#885)

* directory: add google and okta

* add onelogin

* add directory provider

* initialize before sync, upate google provider, remove dead code

* add azure provider

* fix azure provider

* fix gitlab

* add gitlab test, fix azure test

* hook up okta

* remove dead code

* fix tests

* fix flaky test

* remove log line

* only redirect when no session id exists

* prepare rego query as part of create

* return on ctx done

* retry on disconnect for sync

* move jwt signing

* use !=

* use parent ctx for wait

* remove session state, remove logs

* rename function

* add log message

* pre-allocate slice

* use errgroup

* return nil on eof for sync

* move check

* disable timeout on gRPC requests in envoy

* fix dashboard

* delete session on logout

* permanently delete sessions once they are marked as deleted

* remove permanent delete

* fix tests

* remove cache service

* remove kv

* remove refresh docs

* remove obsolete cache docs

* add databroker url option

* cache: use memberlist to detect multiple instances

* add databroker service url

* remove cache service

* remove kv

* remove refresh docs

* remove obsolete cache docs

* add databroker url option

* cache: use memberlist to detect multiple instances

* add databroker service url

* wip

* remove groups and refresh test

* fix redirect, signout

* remove databroker client from proxy

* remove unused method

* remove user dashboard test

* handle missing session ids

* session: reject sessions with no id

* sessions: invalidate old sessions via databroker server version (#930)

* session: add a version field tied to the databroker server version that can be used to invalidate sessions

* fix tests

* add log

* authenticate: create user record immediately, call "get" directly in authorize (#931)

cache/memberlist.go

@@ -0,0 +1,73 @@
+package cache
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	stdlog "log"
+
+	"github.com/hashicorp/memberlist"
+	"github.com/rs/zerolog"
+
+	"github.com/pomerium/pomerium/internal/log"
+)
+
+type memberlistHandler struct {
+	cfg        *memberlist.Config
+	memberlist *memberlist.Memberlist
+	log        zerolog.Logger
+}
+
+func (c *Cache) runMemberList(ctx context.Context) error {
+	mh := new(memberlistHandler)
+	mh.log = log.With().Str("service", "memberlist").Logger()
+
+	pr, pw := io.Pipe()
+	defer pw.Close()
+	defer pr.Close()
+
+	mh.cfg = memberlist.DefaultLANConfig()
+	mh.cfg.Events = mh
+	mh.cfg.Logger = stdlog.New(pw, "", 0)
+	go mh.runLogHandler(pr)
+
+	var err error
+	mh.memberlist, err = memberlist.Create(mh.cfg)
+	if err != nil {
+		return fmt.Errorf("memberlist: error creating memberlist: %w", err)
+	}
+
+	// the only way memberlist would be empty here, following create is if
+	// the current node suddenly died. Still, we check to be safe.
+	if len(mh.memberlist.Members()) == 0 {
+		return errors.New("memberlist: can't find self")
+	}
+
+	<-ctx.Done()
+	return mh.memberlist.Shutdown()
+}
+
+func (mh *memberlistHandler) NotifyJoin(node *memberlist.Node) {
+	mh.log.Debug().Interface("node", node).Msg("node joined")
+
+	if mh.memberlist != nil && len(mh.memberlist.Members()) > 1 {
+		mh.log.Error().Msg("detected multiple cache servers, which is not supported")
+	}
+}
+
+func (mh *memberlistHandler) NotifyLeave(node *memberlist.Node) {
+	mh.log.Debug().Interface("node", node).Msg("node left")
+}
+
+func (mh *memberlistHandler) NotifyUpdate(node *memberlist.Node) {
+	mh.log.Debug().Interface("node", node).Msg("node updated")
+}
+
+func (mh *memberlistHandler) runLogHandler(r io.Reader) {
+	s := bufio.NewScanner(r)
+	for s.Scan() {
+		mh.log.Debug().Msg(s.Text())
+	}
+}