integration: kubernetes support (#2536)

.github/workflows/test.yaml

@@ -100,7 +100,7 @@ jobs:
       matrix:
         go-version: [1.17.x]
         platform: [ubuntu-latest]
-        deployment: [multi, nginx, single, traefik]
+        deployment: [kubernetes, multi, nginx, single, traefik]
         idp: [auth0, azure, github, gitlab, google, oidc, okta, onelogin, ping]
     runs-on: ${{ matrix.platform }}
     steps:

integration/tpl/backends/k3s.libsonnet

@@ -0,0 +1,144 @@
+local utils = import '../utils.libsonnet';
+
+local Command() =
+  [
+    'sh',
+    '-c',
+    |||
+      set -x
+      # the dev image is only available locally, so load it first
+      if [ "${POMERIUM_TAG:-master}" = "dev" ]; then
+        sh -c '
+          while true ; do
+            ctr --connect-timeout=1s --timeout=60s images import /k3s-tmp/pomerium-dev.tar && break
+            sleep 1
+          done
+        ' &
+      fi
+      k3s "$$@"
+    |||,
+    'k3s',
+  ];
+
+local InstallManifest(manifest) =
+  std.join('\n', [
+    'cat <<-END_OF_MANIFEST | tee /tmp/manifest.json',
+    std.manifestJsonEx(manifest, '  '),
+    'END_OF_MANIFEST',
+    'kubectl apply -f /tmp/manifest.json',
+  ] + if manifest.kind == 'Deployment' then [
+    'kubectl wait --for=condition=available deployment/' + manifest.metadata.name,
+  ] else []);
+
+function(idp, manifests) {
+  compose: {
+    services:
+      utils.ComposeService('k3s-server', {
+        image: 'rancher/k3s:${K3S_TAG:-latest}',
+        entrypoint: Command() + [
+          'server',
+          '--disable',
+          'traefik',
+          '--disable',
+          'metrics-server',
+          '--kube-apiserver-arg',
+          'service-node-port-range=1-65535',
+        ],
+        tmpfs: ['/run', '/var/run'],
+        ulimits: {
+          nproc: 65535,
+          nofile: {
+            soft: 65535,
+            hard: 65535,
+          },
+        },
+        privileged: true,
+        restart: 'always',
+        environment: {
+          K3S_TOKEN: 'TOKEN',
+          K3S_KUBECONFIG_OUTPUT: '/k3s-tmp/kubeconfig.yaml',
+          K3S_KUBECONFIG_MODE: '666',
+        },
+        healthcheck: {
+          test: ['CMD', 'kubectl', 'cluster-info'],
+        },
+        ports: [
+          '6443:6443/tcp',
+          '5443:5443/tcp',
+          '443:443/tcp',
+          '80:80/tcp',
+        ],
+        volumes: [
+          'k3s-tmp:/k3s-tmp',
+        ],
+      }) +
+      utils.ComposeService('k3s-agent', {
+        image: 'rancher/k3s:${K3S_TAG:-latest}',
+        entrypoint: Command() + ['agent'],
+        tmpfs: ['/run', '/var/run'],
+        ulimits: {
+          nproc: 65535,
+          nofile: {
+            soft: 65535,
+            hard: 65535,
+          },
+        },
+        privileged: true,
+        restart: 'always',
+        environment: {
+          K3S_URL: 'https://k3s-server:6443',
+          K3S_TOKEN: 'TOKEN',
+        },
+        volumes: [
+          'k3s-tmp:/k3s-tmp',
+        ],
+      }) +
+      utils.ComposeService('k3s-init', {
+        image: 'rancher/k3s:${K3S_TAG:-latest}',
+        depends_on: {
+          'k3s-server': {
+            condition: 'service_healthy',
+          },
+        },
+        entrypoint: [
+          'sh',
+          '-c',
+          |||
+            cat /k3s-tmp/kubeconfig.yaml | sed s/127.0.0.1/k3s-server/g >/tmp/kubeconfig.yaml
+            export KUBECONFIG=/tmp/kubeconfig.yaml
+          ||| + std.join('\n', std.map(
+            InstallManifest,
+            std.sort(manifests, function(manifest) manifest.kind + '/' + manifest.metadata.name)
+          )) + '\n' +
+          |||
+            sleep 30
+          |||,
+        ],
+        volumes: [
+          'k3s-tmp:/k3s-tmp',
+        ],
+      }) +
+      utils.ComposeService('k3s-ready', {
+        depends_on: {
+          'k3s-init': {
+            condition: 'service_completed_successfully',
+          },
+        },
+        image: 'busybox:latest',
+        command: [
+          'sh',
+          '-c',
+          'exit 0',
+        ],
+      }),
+    volumes: {
+      'k3s-tmp': {
+        driver_opts: {
+          type: 'none',
+          device: '/tmp',
+          o: 'bind',
+        },
+      },
+    },
+  },
+}

integration/tpl/clusters/auth0-kubernetes/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/kubernetes.libsonnet')('auth0')

integration/tpl/clusters/auth0-multi/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/multi.libsonnet')('auth0')

integration/tpl/clusters/azure-kubernetes/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/kubernetes.libsonnet')('azure')

integration/tpl/clusters/azure-multi/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/multi.libsonnet')('azure')

integration/tpl/clusters/github-kubernetes/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/kubernetes.libsonnet')('github')

integration/tpl/clusters/github-multi/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/multi.libsonnet')('github')

integration/tpl/clusters/gitlab-kubernetes/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/kubernetes.libsonnet')('gitlab')

integration/tpl/clusters/gitlab-multi/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/multi.libsonnet')('gitlab')

integration/tpl/clusters/google-kubernetes/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/kubernetes.libsonnet')('google')

integration/tpl/clusters/google-multi/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/multi.libsonnet')('google')

integration/tpl/clusters/oidc-kubernetes/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/kubernetes.libsonnet')('oidc')

integration/tpl/clusters/oidc-multi/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/multi.libsonnet')('oidc')

integration/tpl/clusters/okta-kubernetes/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/kubernetes.libsonnet')('okta')

integration/tpl/clusters/okta-multi/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/multi.libsonnet')('okta')

integration/tpl/clusters/onelogin-kubernetes/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/kubernetes.libsonnet')('onelogin')

integration/tpl/clusters/onelogin-multi/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/multi.libsonnet')('onelogin')

integration/tpl/clusters/ping-kubernetes/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/kubernetes.libsonnet')('ping')

integration/tpl/clusters/ping-multi/compose.yml.jsonnet

@@ -0,0 +1 @@
+(import '../../deployments/multi.libsonnet')('ping')

integration/tpl/deployments/kubernetes.libsonnet

@@ -0,0 +1,19 @@
+local utils = import '../utils.libsonnet';
+
+function(idp) utils.Merge([
+  (import '../backends/k3s.libsonnet')(
+    idp,
+    (import '../backends/fortio.libsonnet')().kubernetes +
+    (import '../backends/httpdetails.libsonnet')().kubernetes +
+    (import '../backends/mock-idp.libsonnet')(idp).kubernetes +
+    (import '../backends/pomerium.libsonnet')('single', idp, '.default.svc.cluster.local').kubernetes +
+    (import '../backends/redis.libsonnet')().kubernetes +
+    (import '../backends/verify.libsonnet')('single').kubernetes +
+    (import '../backends/websocket-echo.libsonnet')().kubernetes
+  ).compose,
+  {
+    networks: {
+      main: {},
+    },
+  },
+])

scripts/build-dev-docker.bash

@@ -24,4 +24,6 @@ ENTRYPOINT [ "/bin/pomerium" ]
 CMD ["-config","/pomerium/config.yaml"]
 EOF
   docker build --tag=pomerium/pomerium:dev .
+  # save the image so we can load it from docker-compose
+  docker save --output=/tmp/pomerium-dev.tar pomerium/pomerium:dev
 )