Skip to content

Commit 2ada6f8

Browse files
pkosiecpkosiec
authored
Implement TypeInstance Value Resolver and modify TypeInstance upload (#697)
1 parent c0d2515 commit 2ada6f8

30 files changed

+1306
-65
lines changed

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ all: generate build-all-images test-unit test-lint ## Default: generate all, bui
1717
# Building #
1818
############
1919

20-
APPS = gateway k8s-engine hub-js argo-runner helm-runner cloudsql-runner populator terraform-runner argo-actions gitlab-api-runner secret-storage-backend helm-storage-backend
20+
APPS = gateway k8s-engine hub-js argo-runner helm-runner cloudsql-runner populator terraform-runner argo-actions gitlab-api-runner secret-storage-backend helm-storage-backend ti-value-fetcher
2121
TESTS = e2e local-hub
2222
INFRA = json-go-gen graphql-schema-linter jinja2 merger
2323

‎cmd/argo-actions/README.md

+27-3
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,44 @@ argo-actions is intended to be run as an Argo Workflow step which downloads, upl
1313
## Prerequisites
1414

1515
- [Go](https://golang.org)
16+
- [Local Capact development cluster](https://capact.io/community/development/development-guide#development-cluster) created with the `USE_TEST_SETUP=true make dev-cluster` command
1617

17-
### Running
18+
## Usage
1819

19-
To run it locally you need to enable port forwarding for the Local Hub:
20+
To run it locally you need to enable port forwarding for the Local and Public Hub:
2021
```bash
2122
kubectl -n capact-system port-forward svc/capact-hub-local --address 0.0.0.0 8888:80
23+
kubectl -n capact-system port-forward svc/capact-hub-public --address 0.0.0.0 8890:80
2224
```
2325

26+
### Download
27+
2428
For downloading at least one Type Instance needs to exist. Passing structs using environment variables looks like this: {field1,field2}. For example APP_DOWNLOAD_CONFIG="{ID,path}"
2529

2630
```bash
27-
APP_ACTION=DownloadAction APP_DOWNLOAD_CONFIG="{2282814e-7571-4708-9279-717aea3c6d08,/tmp/action.yaml}" APP_LOCAL_HUB_ENDPOINT=http://localhost:8888/graphql ./argo-actions
31+
APP_ACTION=DownloadAction APP_DOWNLOAD_CONFIG="{2282814e-7571-4708-9279-717aea3c6d08,/tmp/action.yaml}" APP_LOCAL_HUB_ENDPOINT=http://localhost:8888/graphql go run cmd/argo-actions/main.go
2832
```
2933

34+
### Upload
35+
36+
To upload example TypeInstances:
37+
38+
1. Install [Helm Storage Backends](https://capact.io/docs/feature/storage-backends/helm) and use them in Policy
39+
2. Install PostgreSQL according to the [Helm Runner installation](../helm-runner/README.md#installation) section.
40+
3. In the [`payload.yaml`](./example-input/upload/payload.yaml) file:
41+
- replace `{helm release storage backend ID}` with the ID of the installed Helm Release backend
42+
- replace `{test storage backend ID}` with the ID of the installed Helm Release backend
43+
4. Run:
44+
45+
```bash
46+
APP_ACTION=UploadAction \
47+
APP_UPLOAD_CONFIG_PAYLOAD_FILEPATH=cmd/argo-actions/example-input/upload/payload.yaml \
48+
APP_UPLOAD_CONFIG_TYPE_INSTANCES_DIR=cmd/argo-actions/example-input/upload/typeinstances \
49+
APP_LOCAL_HUB_ENDPOINT=http://localhost:8888/graphql \
50+
APP_PUBLIC_HUB_ENDPOINT=http://localhost:8890/graphql \
51+
go run cmd/argo-actions/main.go
52+
```
53+
3054
## Configuration
3155

3256
The following environment variables can be set:

‎cmd/argo-actions/example-input/upload/payload.yaml

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
typeInstances:
2+
- alias: context
3+
attributes: []
4+
backend:
5+
context: null
6+
id: "{helm release storage backend ID}"
7+
createdBy: default/example
8+
typeRef:
9+
path: cap.type.helm.chart.release
10+
revision: 0.1.0
11+
value: null
12+
- alias: value
13+
attributes: []
14+
createdBy: default/example
15+
typeRef:
16+
path: cap.type.capactio.capact.validation.single-key
17+
revision: 0.1.0
18+
value: null
19+
- alias: value-context
20+
attributes: []
21+
createdBy: default/example
22+
typeRef:
23+
path: cap.type.capactio.capact.validation.single-key
24+
revision: 0.1.0
25+
backend:
26+
context: null
27+
id: "{test storage backend ID}"
28+
value: null
29+
usesRelations:
30+
- from: context
31+
to: value
32+
- from: context
33+
to: value-context

‎cmd/argo-actions/example-input/upload/typeinstances/context

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
backend:
2+
context:
3+
chartLocation: https://charts.bitnami.com/bitnami
4+
driver: secrets
5+
name: example-release
6+
namespace: default
7+
value:
8+
chart:
9+
name: postgresql
10+
repo: https://charts.bitnami.com/bitnami
11+
version: 10.2.5
12+
name: example-release
13+
namespace: default

‎cmd/argo-actions/example-input/upload/typeinstances/value

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
value:
2+
key: foo

‎cmd/argo-actions/example-input/upload/typeinstances/value-context

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
value:
2+
key: foo
3+
backend:
4+
context:
5+
provider: dotenv

‎cmd/ti-value-fetcher/README.md

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# TypeInstance Value Fetcher
2+
3+
## Overview
4+
5+
TypeInstance Value Fetcher fetches the static value of a TypeInstance artifact based on context.
6+
This helper app is useful inside a Capact Implementation workflow, where a given Argo artifact contains context, based on which the [ContextStorageBackend](../../hub-js/proto/storage_backend.proto) returns a static value.
7+
8+
However, the static value might be needed inside the same workflow, before uploading such artifact as a TypeInstance. This app calls external storage backend and executes `GetPreCreateValue` method to get such value.
9+
10+
## Prerequisites
11+
12+
- [Go](https://golang.org)
13+
- Running Kubernetes cluster
14+
15+
## Usage
16+
17+
1. Run Kubernetes cluster
18+
2. Install PostgreSQL chart according to the [Helm runner installation](../helm-runner/README.md#installation) section.
19+
3. Run the Helm Release storage backend according to the [Helm Release usage](../helm-storage-backend/README.md#helm-release-storage-backend) instruction
20+
4. Run the app:
21+
22+
```bash
23+
APP_LOGGER_DEV_MODE=true \
24+
APP_INPUT_TI_FILE_PATH="cmd/ti-value-fetcher/example-input/input-ti.yaml" \
25+
APP_INPUT_BACKEND_TI_FILE_PATH="cmd/ti-value-fetcher/example-input/storage-backend.yaml" \
26+
go run cmd/ti-value-fetcher/main.go
27+
```
28+
29+
5. See the output:
30+
31+
```bash
32+
cat /tmp/output.yaml
33+
```
34+
35+
## Configuration
36+
37+
| Name | Required | Default | Description |
38+
|--------------------------------|----------|--------------------------|--------------------------------------------------------------------------------------------------------|
39+
| APP_INPUT_TI_FILE_PATH | no | `/tmp/typeinstance.yaml` | The path to the file with TypeInstance artifact, which should have `value` property resolved. |
40+
| APP_INPUT_BACKEND_TI_FILE_PATH | no | `/tmp/backend.yaml` | The path to the Storage Backend TypeInstance artifact with connection details to the external service. |
41+
| APP_OUTPUT_FILE_PATH | no | `/tmp/output.yaml` | The path where the output file is saved. |
42+
| APP_LOGGER_DEV_MODE | no | `false` | Enable development mode logging. |
43+
44+
To configure providers, use environmental variables described in
45+
the [Providers](https://github.com/SpectralOps/teller#providers) paragraph for Teller's Readme.
46+
47+
## Development
48+
49+
To read more about development, see the [Development guide](https://capact.io/community/development/development-guide).

‎cmd/ti-value-fetcher/example-input/input-ti.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
backend:
2+
context:
3+
chartLocation: https://charts.bitnami.com/bitnami
4+
driver: secrets
5+
name: example-release
6+
namespace: default
7+
value: null

‎cmd/ti-value-fetcher/example-input/storage-backend.yaml

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
value:
2+
acceptValue: false
3+
contextSchema:
4+
$schema: http://json-schema.org/draft-07/schema
5+
additionalProperties: false
6+
properties:
7+
chartLocation:
8+
$id: '#/properties/context/properties/chartLocation'
9+
type: string
10+
driver:
11+
$id: '#/properties/context/properties/driver'
12+
default: secrets
13+
enum:
14+
- secrets
15+
- configmaps
16+
- sql
17+
type: string
18+
name:
19+
$id: '#/properties/context/properties/name'
20+
type: string
21+
namespace:
22+
$id: '#/properties/context/properties/namespace'
23+
type: string
24+
required:
25+
- name
26+
- namespace
27+
- chartLocation
28+
type: object
29+
url: localhost:50051

‎cmd/ti-value-fetcher/main.go

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"log"
6+
7+
"google.golang.org/grpc"
8+
9+
"capact.io/capact/internal/logger"
10+
tivaluefetcher "capact.io/capact/internal/ti-value-fetcher"
11+
"github.com/vrischmann/envconfig"
12+
"sigs.k8s.io/controller-runtime/pkg/manager/signals"
13+
)
14+
15+
// Config holds TypeInstance Value resolver configuration.
16+
type Config struct {
17+
Input struct {
18+
TIFilePath string `envconfig:"default=/tmp/input-ti.yaml"`
19+
BackendTIFilePath string `envconfig:"default=/tmp/storage-backend.yaml"`
20+
}
21+
OutputFilePath string `envconfig:"default=/tmp/output.yaml"`
22+
23+
Logger logger.Config
24+
}
25+
26+
const appName = "ti-value-fetcher"
27+
28+
func main() {
29+
var cfg Config
30+
err := envconfig.InitWithPrefix(&cfg, "APP")
31+
exitOnError(err, "while loading configuration")
32+
33+
ctx := signals.SetupSignalHandler()
34+
35+
// setup logger
36+
unnamedLogger, err := logger.New(cfg.Logger)
37+
exitOnError(err, "while creating zap logger")
38+
39+
logger := unnamedLogger.Named(appName)
40+
41+
tiValueFetcher := tivaluefetcher.New(logger)
42+
43+
tiArtifact, storageBackendValue, err := tiValueFetcher.LoadFromFile(cfg.Input.TIFilePath, cfg.Input.BackendTIFilePath)
44+
exitOnError(err, "while loading input files")
45+
46+
res, err := tiValueFetcher.Do(ctx, tiArtifact, storageBackendValue, grpc.WithInsecure())
47+
exitOnError(err, "while resolving TI value")
48+
49+
err = tiValueFetcher.SaveToFile(cfg.OutputFilePath, res)
50+
exitOnError(err, fmt.Sprintf("while saving output to file %q", cfg.OutputFilePath))
51+
}
52+
53+
func exitOnError(err error, context string) {
54+
if err != nil {
55+
log.Fatalf("%s: %v", context, err)
56+
}
57+
}

‎hack/ci/setup-env.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ fi
3737

3838
# TODO: Read components to build in automated way, e.g. from directory structure
3939
cat <<EOT >>"$GITHUB_ENV"
40-
APPS=name=matrix::{"include":[{"APP":"gateway"},{"APP":"k8s-engine"},{"APP":"hub-js"},{"APP":"argo-runner"},{"APP":"helm-runner"},{"APP":"populator"},{"APP":"terraform-runner"},{"APP":"argo-actions"},{"APP":"gitlab-api-runner"},{"APP":"secret-storage-backend"},{"APP":"helm-storage-backend"}]}
40+
APPS=name=matrix::{"include":[{"APP":"gateway"},{"APP":"k8s-engine"},{"APP":"hub-js"},{"APP":"argo-runner"},{"APP":"helm-runner"},{"APP":"populator"},{"APP":"terraform-runner"},{"APP":"argo-actions"},{"APP":"gitlab-api-runner"},{"APP":"secret-storage-backend"},{"APP":"helm-storage-backend"},{"APP":"ti-value-fetcher"}]}
4141
TESTS=name=matrix::{"include":[{"TEST":"e2e"}, {"TEST":"local-hub"}]}
4242
INFRAS=name=matrix::{"include":[{"INFRA":"json-go-gen"},{"INFRA":"graphql-schema-linter"},{"INFRA":"jinja2"},{"INFRA":"merger"}]}
4343
EOT

‎internal/cli/testing/storage_backend.go

+3-7
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"context"
55
"encoding/json"
66

7+
storagebackend "capact.io/capact/pkg/hub/storage-backend"
8+
79
"capact.io/capact/internal/logger"
810
"capact.io/capact/internal/ptr"
911
hublocalgraphql "capact.io/capact/pkg/hub/api/graphql/local"
@@ -43,12 +45,6 @@ const testStorageTypeContextSchema = `
4345
}
4446
`
4547

46-
type typeInstanceValue struct {
47-
URL string `json:"url"`
48-
AcceptValue bool `json:"acceptValue"`
49-
ContextSchema interface{} `json:"contextSchema"`
50-
}
51-
5248
// StorageBackendRegister provides functionality to produce and upload test storage backend TypeInstance.
5349
type StorageBackendRegister struct {
5450
logger *zap.Logger
@@ -91,7 +87,7 @@ func (i *StorageBackendRegister) RegisterTypeInstances(ctx context.Context) erro
9187
Path: testStorageTypePath,
9288
Revision: testStorageTypeRevision,
9389
},
94-
Value: typeInstanceValue{
90+
Value: storagebackend.TypeInstanceValue{
9591
URL: i.cfg.TestStorageBackendURL,
9692
AcceptValue: true,
9793
ContextSchema: contextSchema,

‎internal/helm-storage-backend/helm_rel_fetcher.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ func NewHelmReleaseFetcher(flags *genericclioptions.ConfigFlags) *HelmReleaseFet
4242
}
4343

4444
// FetchHelmRelease returns a given Helm release. It already handles the gRPC errors properly.
45-
func (f *HelmReleaseFetcher) FetchHelmRelease(tiID string, helmRelease HelmRelease) (*release.Release, error) {
45+
func (f *HelmReleaseFetcher) FetchHelmRelease(helmRelease HelmRelease, additionalErrMsg *string) (*release.Release, error) {
4646
cfg, err := f.actionConfigurationProducer(f.helmCfgFlags, *helmRelease.Driver, helmRelease.Namespace)
4747
if err != nil {
4848
return nil, gRPCInternalError(errors.Wrap(err, "while creating Helm get release client"))
@@ -58,7 +58,11 @@ func (f *HelmReleaseFetcher) FetchHelmRelease(tiID string, helmRelease HelmRelea
5858
switch {
5959
case err == nil:
6060
case errors.Is(err, driver.ErrReleaseNotFound):
61-
return nil, status.Error(codes.NotFound, fmt.Sprintf("Helm release '%s/%s' for TypeInstance '%s' was not found", helmRelease.Namespace, helmRelease.Name, tiID))
61+
var additionalErrCtx string
62+
if additionalErrMsg != nil {
63+
additionalErrCtx = fmt.Sprintf(" (%s)", *additionalErrMsg)
64+
}
65+
return nil, status.Error(codes.NotFound, fmt.Sprintf("Helm release '%s/%s'%s was not found", helmRelease.Namespace, helmRelease.Name, additionalErrCtx))
6266
default:
6367
return nil, gRPCInternalError(errors.Wrap(err, "while fetching Helm release"))
6468
}

0 commit comments

Comments
 (0)