Skip to content

Commit

Permalink
feat: envoy control plane
Browse files Browse the repository at this point in the history
  • Loading branch information
FemiNoviaLina committed Oct 21, 2024
1 parent da5b1fa commit 05d7fec
Show file tree
Hide file tree
Showing 15 changed files with 928 additions and 31 deletions.
45 changes: 24 additions & 21 deletions cmd/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,31 +179,34 @@ func StartServer(logger *log.Zap, cfg *config.Shield) error {
}

// serving proxies
cbs, cps, err := serveProxies(ctx, logger, cfg.App.IdentityProxyHeader, cfg.App.UserIDHeader, cfg.Proxy, pgRuleRepository, deps.ResourceService, deps.RelationService, deps.UserService, deps.GroupService, deps.ProjectService, deps.RelationAdapter)
if err != nil {
return err
}
defer func() {
// clean up stage
logger.Info("cleaning up rules proxy blob")
for _, f := range cbs {
if err := f(); err != nil {
logger.Warn("error occurred during shutdown rules proxy blob storages", "err", err)
}
if cfg.Proxy.EnvoyAgent.XDS.Host != "" && cfg.Proxy.EnvoyAgent.XDS.Port != 0 {
serveXDS(ctx, logger, cfg.Proxy, pgRuleRepository)
} else {
cbs, cps, err := serveProxies(ctx, logger, cfg.App.IdentityProxyHeader, cfg.App.UserIDHeader, cfg.Proxy, pgRuleRepository, deps.ResourceService, deps.RelationService, deps.UserService, deps.GroupService, deps.ProjectService, deps.RelationAdapter)
if err != nil {
return err
}
defer func() {
// clean up stage
logger.Info("cleaning up rules proxy blob")
for _, f := range cbs {
if err := f(); err != nil {
logger.Warn("error occurred during shutdown rules proxy blob storages", "err", err)
}
}

logger.Info("cleaning up proxies")
for _, f := range cps {
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), time.Second*20)
if err := f(shutdownCtx); err != nil {
logger.Info("cleaning up proxies")
for _, f := range cps {
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), time.Second*20)
if err := f(shutdownCtx); err != nil {
shutdownCancel()
logger.Warn("error occurred during shutdown proxies", "err", err)
continue
}
shutdownCancel()
logger.Warn("error occurred during shutdown proxies", "err", err)
continue
}
shutdownCancel()
}
}()

}()
}
// serving server
return server.Serve(ctx, logger, cfg.App, nrApp, deps)
}
Expand Down
38 changes: 38 additions & 0 deletions cmd/serve_proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ import (
"github.com/goto/shield/internal/adapter"
"github.com/goto/shield/internal/api/v1beta1"
"github.com/goto/shield/internal/proxy"
"github.com/goto/shield/internal/proxy/envoy/xds"
"github.com/goto/shield/internal/proxy/envoy/xds/ads"
"github.com/goto/shield/internal/proxy/hook"
authz_hook "github.com/goto/shield/internal/proxy/hook/authz"
"github.com/goto/shield/internal/proxy/middleware/attributes"
Expand Down Expand Up @@ -98,6 +100,42 @@ func serveProxies(
return cleanUpBlobs, cleanUpProxies, nil
}

func serveXDS(ctx context.Context, logger *log.Zap, cfg proxy.ServicesConfig, pgRuleRepository *postgres.RuleRepository) ([]func() error, error) {
var cleanUpBlobs []func() error

repositories := make(map[string]ads.Repository)
for _, svcConfig := range cfg.Services {
parsedRuleConfigURL, err := url.Parse(svcConfig.RulesPath)
if err != nil {
return nil, err
}

var repository ads.Repository
switch parsedRuleConfigURL.Scheme {
case rule.RULES_CONFIG_STORAGE_PG:
repository = pgRuleRepository
case rule.RULES_CONFIG_STORAGE_GS,
rule.RULES_CONFIG_STORAGE_FILE,
rule.RULES_CONFIG_STORAGE_MEM:
ruleBlobFS, err := blob.NewStore(ctx, svcConfig.RulesPath, svcConfig.RulesPathSecret)
if err != nil {
return nil, err
}

blobRuleRepository := blob.NewRuleRepository(logger, ruleBlobFS)
if err := blobRuleRepository.InitCache(ctx, ruleCacheRefreshDelay); err != nil {
return nil, err
}
cleanUpBlobs = append(cleanUpBlobs, blobRuleRepository.Close)
repository = blobRuleRepository
default:
return nil, errors.New("invalid rule config storage")
}
repositories[svcConfig.Name] = repository
}
return cleanUpBlobs, xds.Serve(ctx, logger, cfg, repositories)
}

func buildHookPipeline(
log log.Logger,
resourceService v1beta1.ResourceService,
Expand Down
12 changes: 8 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -54,18 +54,21 @@ require (
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
golang.org/x/net v0.27.0
golang.org/x/oauth2 v0.21.0
google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8
google.golang.org/grpc v1.64.0
google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157
google.golang.org/grpc v1.65.0
google.golang.org/protobuf v1.34.2
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
)

require (
cel.dev/expr v0.15.0 // indirect
github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/oklog/run v1.1.0 // indirect
github.com/planetscale/vtprotobuf v0.6.0 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/samber/lo v1.39.0 // indirect
)
Expand Down Expand Up @@ -97,6 +100,7 @@ require (
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/envoyproxy/go-control-plane v0.13.0
github.com/fatih/color v1.17.0 // indirect
github.com/felixge/fgprof v0.9.3 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
Expand All @@ -105,7 +109,7 @@ require (
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-sql-driver/mysql v1.7.1 // indirect
github.com/golang/glog v1.2.0 // indirect
github.com/golang/glog v1.2.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/cel-go v0.13.0 // indirect
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
Expand Down
15 changes: 15 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM=
cel.dev/expr v0.15.0 h1:O1jzfJCQBfL5BFoYktaxwIhuttaQPsVWerH9/EEKx0w=
cel.dev/expr v0.15.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
Expand Down Expand Up @@ -621,6 +623,7 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g=
github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
Expand Down Expand Up @@ -664,6 +667,8 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw=
github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
Expand Down Expand Up @@ -880,6 +885,8 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.
github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ=
github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34=
github.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmdxpwC51i202les=
github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo=
github.com/envoyproxy/protoc-gen-validate v0.6.13/go.mod h1:qEySVqXrEugbHKvmhI8ZqtQi75/RHSSRNpffvB4I6Bw=
Expand Down Expand Up @@ -1060,6 +1067,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68=
github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4=
github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
Expand Down Expand Up @@ -1721,6 +1730,8 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ
github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
github.com/planetscale/vtprotobuf v0.6.0 h1:nBeETjudeJ5ZgBHUz1fVHvbqUKnYOXNhsIEabROxmNA=
github.com/planetscale/vtprotobuf v0.6.0/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
Expand Down Expand Up @@ -2872,6 +2883,8 @@ google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJ
google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s=
google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 h1:W5Xj/70xIA4x60O/IFyXivR5MGqblAb8R3w26pnD6No=
google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8/go.mod h1:vPrPUTsDCYxXWjP7clS81mZ6/803D8K4iM9Ma27VKas=
google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw=
google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0=
google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
Expand Down Expand Up @@ -2921,6 +2934,8 @@ google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD
google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY=
google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg=
google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
Expand Down
15 changes: 14 additions & 1 deletion internal/proxy/config.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,20 @@
package proxy

import "time"

type ServicesConfig struct {
Services []Config `yaml:"services" mapstructure:"services"`
EnvoyAgent EnvoyAgent `yaml:"envoy" mapstructure:"envoy"`
Services []Config `yaml:"services" mapstructure:"services"`
}

type EnvoyAgent struct {
XDS XDS `yaml:"xds" mapstructure:"xds"`
}

type XDS struct {
Host string `yaml:"host" mapstructure:"host"`
Port int `yaml:"port" mapstructure:"port"`
RefreshInterval time.Duration `yaml:"refresh_interval" mapstructure:"refresh_interval" default:"60s"`
}

type Config struct {
Expand Down
24 changes: 24 additions & 0 deletions internal/proxy/envoy/xds/ads/ads.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package ads

import (
"context"
"time"

"github.com/goto/shield/core/rule"
)

const (
CLUSTER_TYPE_URL = "type.googleapis.com/envoy.config.cluster.v3.Cluster"
LISTENER_TYPE_URL = "type.googleapis.com/envoy.config.listener.v3.Listener"
ROUTE_CONFIGURATION_TYPE_URL = "type.googleapis.com/envoy.config.route.v3.RouteConfiguration"

HTTP_CONNECTION_MANAGER_TYPE_URL = "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager"
ROUTER_TYPE_URL = "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
URI_TEMPLATE_TYPE_URL = "type.googleapis.com/envoy.extensions.path.match.uri_template.v3.UriTemplateMatchConfig"
STDOUT_LOGGER_TYPE_URL = "type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog"
)

type Repository interface {
GetAll(ctx context.Context) ([]rule.Ruleset, error)
IsUpdated(ctx context.Context, since time.Time) bool
}
26 changes: 26 additions & 0 deletions internal/proxy/envoy/xds/ads/pubsub.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package ads

import "errors"

type Message struct {
NodeID string
VersionInfo string
Nonce string
}

type MessageChan chan Message

var (
ErrChannelClosed = errors.New("can't send message on closed channel")
)

func (m MessageChan) Push(message Message) (err error) {
defer func() {
if recover() != nil {
err = ErrChannelClosed
}
}()

m <- message
return nil
}
Loading

0 comments on commit 05d7fec

Please sign in to comment.