Skip to content

Commit 3dc7aa2

Browse files
orouzorouz
authored
GCP Assets Inventory integration fetcher (#2460)
1 parent fd13059 commit 3dc7aa2

File tree

7 files changed

+386
-21
lines changed

7 files changed

+386
-21
lines changed

‎internal/flavors/assetinventory/strategy.go

+18-1
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,12 @@ import (
3030
"github.com/elastic/cloudbeat/internal/inventory"
3131
"github.com/elastic/cloudbeat/internal/inventory/awsfetcher"
3232
"github.com/elastic/cloudbeat/internal/inventory/azurefetcher"
33+
"github.com/elastic/cloudbeat/internal/inventory/gcpfetcher"
3334
"github.com/elastic/cloudbeat/internal/resources/providers/awslib"
3435
"github.com/elastic/cloudbeat/internal/resources/providers/azurelib"
3536
azure_auth "github.com/elastic/cloudbeat/internal/resources/providers/azurelib/auth"
37+
gcp_auth "github.com/elastic/cloudbeat/internal/resources/providers/gcplib/auth"
38+
gcp_inventory "github.com/elastic/cloudbeat/internal/resources/providers/gcplib/inventory"
3639
)
3740

3841
type Strategy interface {
@@ -54,7 +57,7 @@ func (s *strategy) NewAssetInventory(ctx context.Context, client beat.Client) (i
5457
case config.ProviderAzure:
5558
fetchers, err = s.initAzureFetchers(ctx)
5659
case config.ProviderGCP:
57-
err = fmt.Errorf("GCP branch not implemented")
60+
fetchers, err = s.initGcpFetchers(ctx)
5861
case "":
5962
err = fmt.Errorf("missing config.v1.asset_inventory_provider setting")
6063
default:
@@ -99,6 +102,20 @@ func (s *strategy) initAzureFetchers(_ context.Context) ([]inventory.AssetFetche
99102
return azurefetcher.New(s.logger, provider, azureConfig), nil
100103
}
101104

105+
func (s *strategy) initGcpFetchers(ctx context.Context) ([]inventory.AssetFetcher, error) {
106+
cfgProvider := &gcp_auth.ConfigProvider{AuthProvider: &gcp_auth.GoogleAuthProvider{}}
107+
gcpConfig, err := cfgProvider.GetGcpClientConfig(ctx, s.cfg.CloudConfig.Gcp, s.logger)
108+
if err != nil {
109+
return nil, fmt.Errorf("failed to initialize gcp config: %w", err)
110+
}
111+
inventoryInitializer := &gcp_inventory.ProviderInitializer{}
112+
provider, err := inventoryInitializer.Init(ctx, s.logger, *gcpConfig)
113+
if err != nil {
114+
return nil, fmt.Errorf("failed to initialize gcp asset inventory: %v", err)
115+
}
116+
return gcpfetcher.New(s.logger, provider), nil
117+
}
118+
102119
func GetStrategy(logger *logp.Logger, cfg *config.Config) Strategy {
103120
return &strategy{
104121
logger: logger,

‎internal/inventory/ASSETS.md

+12-12
Original file line numberDiff line numberDiff line change
@@ -162,9 +162,9 @@ Infrastructure: 27% (12/43)
162162

163163
## GCP Resources
164164

165-
**Progress: 0% (0/25)**
166-
Identity: 0% (0/4)
167-
Infrastructure: 0% (0/20)
165+
**Progress: 36% (9/25)**
166+
Identity: 50% (2/4)
167+
Infrastructure: 35% (7/20)
168168
Management: 0% (0/1)
169169

170170
<details> <summary>Full table</summary>
@@ -173,28 +173,28 @@ Management: 0% (0/1)
173173
|---|---|---|---|---|
174174
| Identity | Access Management | IAM Policy | GCP IAM Policy | No ❌ |
175175
| Identity | Access Management | IAM Role | GCP IAM Role | No ❌ |
176-
| Identity | Service Identity | Service Account Key | GCP Service Account Key | No ❌ |
177-
| Identity | Service Identity | Service Account | GCP Service Account | No ❌ |
178-
| Infrastructure | Compute | Virtual Machine | GCP Instance | No ❌ |
176+
| Identity | Service Identity | Service Account Key | GCP Service Account Key | Yes ✅ |
177+
| Identity | Service Identity | Service Account | GCP Service Account | Yes ✅ |
178+
| Infrastructure | Compute | Virtual Machine | GCP Instance | Yes ✅ |
179179
| Infrastructure | Container | Orchestration | GKE Cluster | No ❌ |
180180
| Infrastructure | Container | Serverless | GCP Cloud Run Service | No ❌ |
181-
| Infrastructure | Management | Cloud Account | GCP Organization | No ❌ |
182-
| Infrastructure | Management | Cloud Account | GCP Project | No ❌ |
183-
| Infrastructure | Management | Resource Hierarchy | GCP Folder | No ❌ |
181+
| Infrastructure | Management | Cloud Account | GCP Organization | Yes ✅ |
182+
| Infrastructure | Management | Cloud Account | GCP Project | Yes ✅ |
183+
| Infrastructure | Management | Resource Hierarchy | GCP Folder | Yes ✅ |
184184
| Infrastructure | Network | DNS | GCP DNS Record Set | No ❌ |
185185
| Infrastructure | Network | DNS | GCP DNS Zone | No ❌ |
186186
| Infrastructure | Network | Firewall Rule | GCP IP Rule | No ❌ |
187-
| Infrastructure | Network | Firewall | GCP Firewall | No ❌ |
187+
| Infrastructure | Network | Firewall | GCP Firewall | Yes ✅ |
188188
| Infrastructure | Network | Firewall | GCP Network Tag | No ❌ |
189189
| Infrastructure | Network | IP Address Range | IP Range | No ❌ |
190190
| Infrastructure | Network | Load Balancing | GCP Compute Target Pool | No ❌ |
191191
| Infrastructure | Network | Load Balancing | GCP Forwarding Rule | No ❌ |
192192
| Infrastructure | Network | Network Interface | GCP Network Interface | No ❌ |
193193
| Infrastructure | Network | Network Interface | GCP Network Interface Access Config | No ❌ |
194-
| Infrastructure | Network | Subnet | GCP Subnet | No ❌ |
194+
| Infrastructure | Network | Subnet | GCP Subnet | Yes ✅ |
195195
| Infrastructure | Network | Virtual Network | GCP VPC | No ❌ |
196196
| Infrastructure | Serverless | Function | GCP Cloud Function | No ❌ |
197-
| Infrastructure | Storage | Object Storage | GCP Bucket | No ❌ |
197+
| Infrastructure | Storage | Object Storage | GCP Bucket | Yes ✅ |
198198
| Management | Resource Management | Label | GCP Bucket Label | No ❌ |
199199

200200
</details>

‎internal/inventory/asset.go

+39-8
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ const (
4040
SubCategoryMessaging AssetSubCategory = "messaging"
4141
SubCategoryNetwork AssetSubCategory = "network"
4242
SubCategoryStorage AssetSubCategory = "storage"
43+
SubCategoryServiceIdentity AssetSubCategory = "service-identity"
4344
)
4445

4546
// AssetType is used to build the document index. Use only numbers, letters and dashes (-)
@@ -72,6 +73,9 @@ const (
7273
TypeVirtualMachine AssetType = "virtual-machine"
7374
TypeVirtualNetwork AssetType = "virtual-network"
7475
TypeWebApplication AssetType = "web-application"
76+
TypeServiceAccount AssetType = "service-account"
77+
TypeServiceAccountKey AssetType = "service-account-key"
78+
TypeResourceHierarchy AssetType = "resource-hierarchy"
7579
)
7680

7781
// AssetSubType is used to build the document index. Use only numbers, letters and dashes (-)
@@ -115,11 +119,21 @@ const (
115119
SubTypeVpc AssetSubType = "vpc"
116120
SubTypeVpcAcl AssetSubType = "s3-access-control-list"
117121
SubTypeVpcPeeringConnection AssetSubType = "vpc-peering-connection"
122+
SubTypeGcpProject AssetSubType = "gcp-project"
123+
SubTypeGcpInstance AssetSubType = "gcp-instance"
124+
SubTypeGcpSubnet AssetSubType = "gcp-subnet"
125+
SubTypeGcpFirewall AssetSubType = "gcp-firewall"
126+
SubTypeGcpBucket AssetSubType = "gcp-bucket"
127+
SubTypeGcpOrganization AssetSubType = "gcp-organization"
128+
SubTypeGcpFolder AssetSubType = "gcp-folder"
129+
SubTypeGcpServiceAccount AssetSubType = "gcp-service-account"
130+
SubTypeGcpServiceAccountKey AssetSubType = "gcp-service-account-key"
118131
)
119132

120133
const (
121134
AwsCloudProvider = "aws"
122135
AzureCloudProvider = "azure"
136+
GcpCloudProvider = "gcp"
123137
)
124138

125139
// AssetClassification holds the taxonomy of an asset
@@ -171,6 +185,17 @@ var (
171185
AssetClassificationAzureSubscription = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryManagement, Type: TypeCloudAccount, SubType: SubTypeAzureSubscription}
172186
AssetClassificationAzureTenant = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryManagement, Type: TypeCloudAccount, SubType: SubTypeAzureTenant}
173187
AssetClassificationAzureVirtualMachine = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryCompute, Type: TypeVirtualMachine, SubType: SubTypeAzureVirtualMachine}
188+
189+
// GCP
190+
AssetClassificationGcpProject = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryManagement, Type: TypeCloudAccount, SubType: SubTypeGcpProject}
191+
AssetClassificationGcpOrganization = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryManagement, Type: TypeCloudAccount, SubType: SubTypeGcpOrganization}
192+
AssetClassificationGcpFolder = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryManagement, Type: TypeResourceHierarchy, SubType: SubTypeGcpFolder}
193+
AssetClassificationGcpInstance = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryCompute, Type: TypeVirtualMachine, SubType: SubTypeGcpInstance}
194+
AssetClassificationGcpBucket = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryStorage, Type: TypeObjectStorage, SubType: SubTypeGcpBucket}
195+
AssetClassificationGcpFirewall = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryNetwork, Type: TypeFirewall, SubType: SubTypeGcpFirewall}
196+
AssetClassificationGcpSubnet = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryNetwork, Type: TypeSubnet, SubType: SubTypeGcpSubnet}
197+
AssetClassificationGcpServiceAccount = AssetClassification{Category: CategoryIdentity, SubCategory: SubCategoryServiceIdentity, Type: TypeServiceAccount, SubType: SubTypeGcpServiceAccount}
198+
AssetClassificationGcpServiceAccountKey = AssetClassification{Category: CategoryIdentity, SubCategory: SubCategoryServiceIdentity, Type: TypeServiceAccountKey, SubType: SubTypeGcpServiceAccountKey}
174199
)
175200

176201
// AssetEvent holds the whole asset
@@ -210,21 +235,27 @@ type AssetNetwork struct {
210235

211236
// AssetCloud contains information about the cloud provider
212237
type AssetCloud struct {
213-
AvailabilityZone *string `json:"availability_zone,omitempty"`
214-
Provider string `json:"provider,omitempty"`
215-
Region string `json:"region,omitempty"`
216-
Account AssetCloudAccount `json:"account"`
217-
Instance *AssetCloudInstance `json:"instance,omitempty"`
218-
Machine *AssetCloudMachine `json:"machine,omitempty"`
219-
Project *AssetCloudProject `json:"project,omitempty"`
220-
Service *AssetCloudService `json:"service,omitempty"`
238+
AvailabilityZone *string `json:"availability_zone,omitempty"`
239+
Provider string `json:"provider,omitempty"`
240+
Region string `json:"region,omitempty"`
241+
Account AssetCloudAccount `json:"account"`
242+
Organization AssetCloudOrganization `json:"organization,omitempty"`
243+
Instance *AssetCloudInstance `json:"instance,omitempty"`
244+
Machine *AssetCloudMachine `json:"machine,omitempty"`
245+
Project *AssetCloudProject `json:"project,omitempty"`
246+
Service *AssetCloudService `json:"service,omitempty"`
221247
}
222248

223249
type AssetCloudAccount struct {
224250
Id string `json:"id,omitempty"`
225251
Name string `json:"name,omitempty"`
226252
}
227253

254+
type AssetCloudOrganization struct {
255+
Id string `json:"id,omitempty"`
256+
Name string `json:"name,omitempty"`
257+
}
258+
228259
type AssetCloudInstance struct {
229260
Id string `json:"id,omitempty"`
230261
Name string `json:"name,omitempty"`

‎internal/inventory/gcpfetcher/fetcher_assets.go

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
// Licensed to Elasticsearch B.V. under one or more contributor
2+
// license agreements. See the NOTICE file distributed with
3+
// this work for additional information regarding copyright
4+
// ownership. Elasticsearch B.V. licenses this file to you under
5+
// the Apache License, Version 2.0 (the "License"); you may
6+
// not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package gcpfetcher
19+
20+
import (
21+
"context"
22+
23+
"github.com/elastic/elastic-agent-libs/logp"
24+
25+
"github.com/elastic/cloudbeat/internal/inventory"
26+
gcpinventory "github.com/elastic/cloudbeat/internal/resources/providers/gcplib/inventory"
27+
)
28+
29+
type (
30+
assetsInventory struct {
31+
logger *logp.Logger
32+
provider inventoryProvider
33+
}
34+
inventoryProvider interface {
35+
ListAllAssetTypesByName(ctx context.Context, assets []string) ([]*gcpinventory.ExtendedGcpAsset, error)
36+
}
37+
ResourcesClassification struct {
38+
assetType string
39+
classification inventory.AssetClassification
40+
}
41+
)
42+
43+
var ResourcesToFetch = []ResourcesClassification{
44+
{gcpinventory.CrmOrgAssetType, inventory.AssetClassificationGcpOrganization},
45+
{gcpinventory.CrmFolderAssetType, inventory.AssetClassificationGcpFolder},
46+
{gcpinventory.CrmProjectAssetType, inventory.AssetClassificationGcpProject},
47+
{gcpinventory.ComputeInstanceAssetType, inventory.AssetClassificationGcpInstance},
48+
{gcpinventory.ComputeFirewallAssetType, inventory.AssetClassificationGcpFirewall},
49+
{gcpinventory.StorageBucketAssetType, inventory.AssetClassificationGcpBucket},
50+
{gcpinventory.ComputeSubnetworkAssetType, inventory.AssetClassificationGcpSubnet},
51+
{gcpinventory.IamServiceAccountAssetType, inventory.AssetClassificationGcpServiceAccount},
52+
{gcpinventory.IamServiceAccountKeyAssetType, inventory.AssetClassificationGcpServiceAccountKey},
53+
}
54+
55+
func newAssetsInventoryFetcher(logger *logp.Logger, provider inventoryProvider) inventory.AssetFetcher {
56+
return &assetsInventory{
57+
logger: logger,
58+
provider: provider,
59+
}
60+
}
61+
62+
func (f *assetsInventory) Fetch(ctx context.Context, assetChan chan<- inventory.AssetEvent) {
63+
for _, r := range ResourcesToFetch {
64+
f.fetch(ctx, assetChan, r.assetType, r.classification)
65+
}
66+
}
67+
68+
func (f *assetsInventory) fetch(ctx context.Context, assetChan chan<- inventory.AssetEvent, assetType string, classification inventory.AssetClassification) {
69+
f.logger.Infof("Fetching %s", assetType)
70+
defer f.logger.Infof("Fetching %s - Finished", assetType)
71+
72+
gcpAssets, err := f.provider.ListAllAssetTypesByName(ctx, []string{assetType})
73+
if err != nil {
74+
f.logger.Errorf("Could not fetch %s: %v", assetType, err)
75+
return
76+
}
77+
78+
for _, item := range gcpAssets {
79+
assetChan <- inventory.NewAssetEvent(
80+
classification,
81+
[]string{item.Name},
82+
item.Name,
83+
inventory.WithRawAsset(item),
84+
inventory.WithCloud(inventory.AssetCloud{
85+
Provider: inventory.GcpCloudProvider,
86+
Account: inventory.AssetCloudAccount{
87+
Id: item.CloudAccount.AccountId,
88+
Name: item.CloudAccount.AccountName,
89+
},
90+
Organization: inventory.AssetCloudOrganization{
91+
Id: item.CloudAccount.OrganisationId,
92+
Name: item.CloudAccount.OrganizationName,
93+
},
94+
Service: &inventory.AssetCloudService{
95+
Name: assetType,
96+
},
97+
}),
98+
)
99+
}
100+
}

‎internal/inventory/gcpfetcher/fetcher_assets_test.go

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
// Licensed to Elasticsearch B.V. under one or more contributor
2+
// license agreements. See the NOTICE file distributed with
3+
// this work for additional information regarding copyright
4+
// ownership. Elasticsearch B.V. licenses this file to you under
5+
// the Apache License, Version 2.0 (the "License"); you may
6+
// not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package gcpfetcher
19+
20+
import (
21+
"testing"
22+
23+
"cloud.google.com/go/asset/apiv1/assetpb"
24+
"github.com/elastic/elastic-agent-libs/logp"
25+
"github.com/samber/lo"
26+
"github.com/stretchr/testify/mock"
27+
28+
"github.com/elastic/cloudbeat/internal/inventory"
29+
"github.com/elastic/cloudbeat/internal/inventory/testutil"
30+
"github.com/elastic/cloudbeat/internal/resources/fetching"
31+
gcpinventory "github.com/elastic/cloudbeat/internal/resources/providers/gcplib/inventory"
32+
)
33+
34+
func TestAccountFetcher_Fetch_Assets(t *testing.T) {
35+
logger := logp.NewLogger("gcpfetcher_test")
36+
assets := []*gcpinventory.ExtendedGcpAsset{
37+
{
38+
Asset: &assetpb.Asset{
39+
Name: "/projects/<project UUID>/some_resource", // name is the ID
40+
},
41+
CloudAccount: &fetching.CloudAccountMetadata{
42+
AccountId: "<project UUID>",
43+
AccountName: "<project name>",
44+
OrganisationId: "<org UUID>",
45+
OrganizationName: "<org name>",
46+
},
47+
},
48+
}
49+
50+
expected := lo.Map(ResourcesToFetch, func(r ResourcesClassification, _ int) inventory.AssetEvent {
51+
return inventory.NewAssetEvent(
52+
r.classification,
53+
[]string{"/projects/<project UUID>/some_resource"},
54+
"/projects/<project UUID>/some_resource",
55+
inventory.WithRawAsset(assets[0]),
56+
inventory.WithCloud(inventory.AssetCloud{
57+
Provider: inventory.GcpCloudProvider,
58+
Account: inventory.AssetCloudAccount{
59+
Id: "<project UUID>",
60+
Name: "<project name>",
61+
},
62+
Organization: inventory.AssetCloudOrganization{
63+
Id: "<org UUID>",
64+
Name: "<org name>",
65+
},
66+
Service: &inventory.AssetCloudService{
67+
Name: r.assetType,
68+
},
69+
}),
70+
)
71+
})
72+
73+
provider := newMockInventoryProvider(t)
74+
provider.EXPECT().ListAllAssetTypesByName(mock.Anything, mock.AnythingOfType("[]string")).Return(assets, nil)
75+
fetcher := newAssetsInventoryFetcher(logger, provider)
76+
testutil.CollectResourcesAndMatch(t, fetcher, expected)
77+
}

‎internal/inventory/gcpfetcher/gcpfetchers.go

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Licensed to Elasticsearch B.V. under one or more contributor
2+
// license agreements. See the NOTICE file distributed with
3+
// this work for additional information regarding copyright
4+
// ownership. Elasticsearch B.V. licenses this file to you under
5+
// the Apache License, Version 2.0 (the "License"); you may
6+
// not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package gcpfetcher
19+
20+
import (
21+
"github.com/elastic/elastic-agent-libs/logp"
22+
23+
"github.com/elastic/cloudbeat/internal/inventory"
24+
gcpinventory "github.com/elastic/cloudbeat/internal/resources/providers/gcplib/inventory"
25+
)
26+
27+
func New(logger *logp.Logger, provider gcpinventory.ServiceAPI) []inventory.AssetFetcher {
28+
return []inventory.AssetFetcher{
29+
newAssetsInventoryFetcher(logger, provider),
30+
}
31+
}

0 commit comments

Comments
 (0)