Skip to content

Commit 5ca0e47

Browse files
JeffreyDallasJeffreyDallas
authored
feat: add flags to support aws as storage destination (#1372)
Signed-off-by: Jeffrey Tang <[email protected]>
1 parent 5b5e276 commit 5ca0e47

File tree

9 files changed

+303
-138
lines changed

9 files changed

+303
-138
lines changed

‎.github/workflows/flow-gcs-test.yaml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,9 @@ jobs:
3434
timeout-minutes: 20
3535
runs-on: solo-linux-large
3636
strategy:
37+
fail-fast: false
3738
matrix:
38-
storageType: ["gcs_only", "gcs_and_minio"]
39+
storageType: ["gcs_only", "minio_only", "aws_only"]
3940
steps:
4041
- name: Harden Runner
4142
uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4
@@ -116,6 +117,7 @@ jobs:
116117
.github/workflows/script/gcs_test.sh
117118
118119
- name: Delete Test Directory after Test
120+
if: matrix.storageType == 'gcs_only' || matrix.storageType == 'aws_only'
119121
run: |
120122
gcloud storage rm --recursive gs://${BUCKET_NAME}/${{ steps.jobs.outputs.job_id }} --project=${{ vars.GCP_S3_PROJECT_ID }}
121123
gcloud storage rm --recursive gs://${BACKUP_BUCKET_NAME}/${{ steps.jobs.outputs.job_id }} --project=${{ vars.GCP_S3_PROJECT_ID }}

‎.github/workflows/script/gcs_test.sh

Lines changed: 45 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ else
2626
fi
2727

2828
if [ -z "${STORAGE_TYPE}" ]; then
29-
storageType="gcs_and_minio"
29+
storageType="aws_only"
3030
else
3131
storageType=${STORAGE_TYPE}
3232
fi
@@ -40,44 +40,68 @@ if [ -z "${PREFIX}" ]; then
4040
echo "PREFIX is not set"
4141
else
4242
echo "Using PREFIX: ${PREFIX}"
43-
BUCKET_PREFIX_OPTION="--storage-bucket-prefix ${PREFIX}"
43+
if [ "${storageType}" == "aws_only" ]; then
44+
STORAGE_OPTIONS=(
45+
"--aws-endpoint" "https://storage.googleapis.com"
46+
"--aws-access-key" "${GCS_ACCESS_KEY}"
47+
"--aws-secrets" "${GCS_SECRET_KEY}"
48+
"--aws-bucket" "${streamBucket}"
49+
"--aws-bucket-prefix" "${PREFIX}"
50+
)
51+
elif [ "${storageType}" == "gcs_only" ]; then
52+
STORAGE_OPTIONS=(
53+
"--gcs-endpoint" "https://storage.googleapis.com"
54+
"--gcs-access-key" "${GCS_ACCESS_KEY}"
55+
"--gcs-secrets" "${GCS_SECRET_KEY}"
56+
"--gcs-bucket" "${streamBucket}"
57+
"--gcs-bucket-prefix" "${PREFIX}"
58+
)
59+
fi
60+
61+
if [ "${storageType}" == "aws_only" ] || [ "${storageType}" == "gcs_only" ]; then
62+
MIRROR_STORAGE_OPTIONS=(
63+
"--storage-endpoint" "https://storage.googleapis.com"
64+
"--storage-access-key" "${GCS_ACCESS_KEY}"
65+
"--storage-secrets" "${GCS_SECRET_KEY}"
66+
"--storage-bucket" "${streamBucket}"
67+
"--storage-bucket-prefix" "${PREFIX}"
68+
)
69+
fi
4470
fi
4571

72+
echo "STORAGE_OPTIONS: " "${STORAGE_OPTIONS[@]}"
73+
echo "MIRROR_STORAGE_OPTIONS: " "${MIRROR_STORAGE_OPTIONS[@]}"
74+
4675
echo "${GCP_SERVICE_ACCOUNT_TOKEN}" > gcp_service_account.json
4776

4877
echo "Using bucket name: ${streamBucket}"
4978
echo "Test storage type: ${storageType}"
5079

51-
if [[ -z "${SOLO_TEST_CLUSTER}" && ${SOLO_CLUSTER_NAME} != "" ]]; then
52-
SOLO_CLUSTER_NAME=solo-e2e
53-
else
54-
SOLO_CLUSTER_NAME=${SOLO_TEST_CLUSTER}
55-
fi
80+
SOLO_CLUSTER_NAME=solo-e2e
5681
SOLO_NAMESPACE=solo-e2e
5782
SOLO_CLUSTER_SETUP_NAMESPACE=solo-setup
58-
83+
SOLO_DEPLOYMENT=solo-e2e
5984

6085
kind delete cluster -n "${SOLO_CLUSTER_NAME}"
6186
kind create cluster -n "${SOLO_CLUSTER_NAME}"
6287
npm run solo-test -- init
6388
npm run solo-test -- cluster setup \
6489
-s "${SOLO_CLUSTER_SETUP_NAMESPACE}"
6590
npm run solo-test -- node keys --gossip-keys --tls-keys -i node1
66-
npm run solo-test -- network deploy -i node1 -n "${SOLO_NAMESPACE}" \
67-
--storage-endpoint "https://storage.googleapis.com" \
68-
--storage-access-key "${GCS_ACCESS_KEY}" --storage-secrets "${GCS_SECRET_KEY}" \
69-
--storage-type "${storageType}" --storage-bucket "${streamBucket}" $BUCKET_PREFIX_OPTION \
91+
npm run solo-test -- deployment create -n "${SOLO_NAMESPACE}" --context kind-"${SOLO_CLUSTER_NAME}" --email [email protected] --deployment-clusters kind-"${SOLO_CLUSTER_NAME}" --deployment "${SOLO_DEPLOYMENT}"
92+
npm run solo-test -- network deploy -i node1 --deployment "${SOLO_DEPLOYMENT}" \
93+
--storage-type "${storageType}" \
94+
"${STORAGE_OPTIONS[@]}" \
7095
--backup-bucket "${streamBackupBucket}" \
7196
--google-credential gcp_service_account.json
7297

73-
npm run solo-test -- node setup -i node1 -n "${SOLO_NAMESPACE}"
74-
npm run solo-test -- node start -i node1 -n "${SOLO_NAMESPACE}"
75-
npm run solo-test -- mirror-node deploy --namespace "${SOLO_NAMESPACE}" \
76-
--storage-endpoint "https://storage.googleapis.com" \
77-
--storage-access-key "${GCS_ACCESS_KEY}" --storage-secrets "${GCS_SECRET_KEY}" \
78-
--storage-type "${storageType}" --storage-bucket "${streamBucket}" $BUCKET_PREFIX_OPTION
98+
npm run solo-test -- node setup -i node1 --deployment "${SOLO_DEPLOYMENT}"
99+
npm run solo-test -- node start -i node1 --deployment "${SOLO_DEPLOYMENT}"
100+
npm run solo-test -- mirror-node deploy --deployment "${SOLO_DEPLOYMENT}" \
101+
--storage-type "${storageType}" \
102+
"${MIRROR_STORAGE_OPTIONS[@]}" \
79103

80-
npm run solo-test -- explorer deploy -n "${SOLO_NAMESPACE}" -s "${SOLO_CLUSTER_SETUP_NAMESPACE}"
104+
npm run solo-test -- explorer deploy -s "${SOLO_CLUSTER_SETUP_NAMESPACE}" --deployment "${SOLO_DEPLOYMENT}"
81105

82106
kubectl port-forward -n "${SOLO_NAMESPACE}" svc/haproxy-node1-svc 50211:50211 > /dev/null 2>&1 &
83107

@@ -88,7 +112,7 @@ cd ..; create_test_account ; cd -
88112

89113
node examples/create-topic.js
90114

91-
npm run solo-test -- node stop -i node1 -n "${SOLO_NAMESPACE}"
115+
npm run solo-test -- node stop -i node1 --deployment "${SOLO_DEPLOYMENT}"
92116

93117
echo "Waiting for backup uploader to run"
94118
# manually call script "backup.sh" from container backup-uploader since it only runs every 5 minutes
@@ -102,4 +126,4 @@ if grep -q \""error\"" backup-uploader.log; then
102126
exit 1
103127
fi
104128

105-
npm run solo-test -- network destroy -n "${SOLO_NAMESPACE}" --force -q
129+
npm run solo-test -- network destroy --deployment "${SOLO_DEPLOYMENT}" --force -q

‎src/commands/flags.ts

Lines changed: 149 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1763,66 +1763,125 @@ export class Flags {
17631763
definition: {
17641764
defaultValue: constants.StorageType.MINIO_ONLY,
17651765
describe:
1766-
'storage type for saving stream files, available options are minio_only, gcs_and_minio, s3_only, gcs_only, s3_and_gcs',
1766+
'storage type for saving stream files, available options are minio_only, aws_only, gcs_only, aws_and_gcs',
17671767
type: 'StorageType',
17681768
},
17691769
prompt: undefined,
17701770
};
17711771

1772-
static readonly storageAccessKey: CommandFlag = {
1773-
constName: 'storageAccessKey',
1774-
name: 'storage-access-key',
1772+
static readonly gcsAccessKey: CommandFlag = {
1773+
constName: 'gcsAccessKey',
1774+
name: 'gcs-access-key',
17751775
definition: {
17761776
defaultValue: '',
1777-
describe: 'storage access key',
1777+
describe: 'gcs storage access key',
17781778
type: 'string',
17791779
dataMask: constants.STANDARD_DATAMASK,
17801780
},
17811781
prompt: undefined,
17821782
};
17831783

1784-
static readonly storageSecrets: CommandFlag = {
1785-
constName: 'storageSecrets',
1786-
name: 'storage-secrets',
1784+
static readonly gcsSecrets: CommandFlag = {
1785+
constName: 'gcsSecrets',
1786+
name: 'gcs-secrets',
17871787
definition: {
17881788
defaultValue: '',
1789-
describe: 'storage secret key',
1789+
describe: 'gcs storage secret key',
17901790
type: 'string',
17911791
dataMask: constants.STANDARD_DATAMASK,
17921792
},
17931793
prompt: undefined,
17941794
};
17951795

1796-
static readonly storageEndpoint: CommandFlag = {
1797-
constName: 'storageEndpoint',
1798-
name: 'storage-endpoint',
1796+
static readonly gcsEndpoint: CommandFlag = {
1797+
constName: 'gcsEndpoint',
1798+
name: 'gcs-endpoint',
17991799
definition: {
18001800
defaultValue: '',
1801-
describe: 'storage endpoint URL',
1801+
describe: 'gcs storage endpoint URL',
18021802
type: 'string',
18031803
dataMask: constants.STANDARD_DATAMASK,
18041804
},
18051805
prompt: undefined,
18061806
};
18071807

1808-
static readonly storageBucket: CommandFlag = {
1809-
constName: 'storageBucket',
1810-
name: 'storage-bucket',
1808+
static readonly gcsBucket: CommandFlag = {
1809+
constName: 'gcsBucket',
1810+
name: 'gcs-bucket',
18111811
definition: {
18121812
defaultValue: '',
1813-
describe: 'name of storage bucket',
1813+
describe: 'name of gcs storage bucket',
18141814
type: 'string',
18151815
dataMask: constants.STANDARD_DATAMASK,
18161816
},
18171817
prompt: undefined,
18181818
};
18191819

1820-
static readonly storageBucketPrefix: CommandFlag = {
1821-
constName: 'storageBucketPrefix',
1822-
name: 'storage-bucket-prefix',
1820+
static readonly gcsBucketPrefix: CommandFlag = {
1821+
constName: 'gcsBucketPrefix',
1822+
name: 'gcs-bucket-prefix',
1823+
definition: {
1824+
defaultValue: '',
1825+
describe: 'path prefix of google storage bucket',
1826+
type: 'string',
1827+
},
1828+
prompt: undefined,
1829+
};
1830+
1831+
static readonly awsAccessKey: CommandFlag = {
1832+
constName: 'awsAccessKey',
1833+
name: 'aws-access-key',
1834+
definition: {
1835+
defaultValue: '',
1836+
describe: 'aws storage access key',
1837+
type: 'string',
1838+
dataMask: constants.STANDARD_DATAMASK,
1839+
},
1840+
prompt: undefined,
1841+
};
1842+
1843+
static readonly awsSecrets: CommandFlag = {
1844+
constName: 'awsSecrets',
1845+
name: 'aws-secrets',
1846+
definition: {
1847+
defaultValue: '',
1848+
describe: 'aws storage secret key',
1849+
type: 'string',
1850+
dataMask: constants.STANDARD_DATAMASK,
1851+
},
1852+
prompt: undefined,
1853+
};
1854+
1855+
static readonly awsEndpoint: CommandFlag = {
1856+
constName: 'awsEndpoint',
1857+
name: 'aws-endpoint',
1858+
definition: {
1859+
defaultValue: '',
1860+
describe: 'aws storage endpoint URL',
1861+
type: 'string',
1862+
dataMask: constants.STANDARD_DATAMASK,
1863+
},
1864+
prompt: undefined,
1865+
};
1866+
1867+
static readonly awsBucket: CommandFlag = {
1868+
constName: 'awsBucket',
1869+
name: 'aws-bucket',
1870+
definition: {
1871+
defaultValue: '',
1872+
describe: 'name of aws storage bucket',
1873+
type: 'string',
1874+
dataMask: constants.STANDARD_DATAMASK,
1875+
},
1876+
prompt: undefined,
1877+
};
1878+
1879+
static readonly awsBucketPrefix: CommandFlag = {
1880+
constName: 'awsBucketPrefix',
1881+
name: 'aws-bucket-prefix',
18231882
definition: {
18241883
defaultValue: '',
1825-
describe: 'path prefix of storage bucket',
1884+
describe: 'path prefix of aws storage bucket',
18261885
type: 'string',
18271886
},
18281887
prompt: undefined,
@@ -1852,6 +1911,65 @@ export class Flags {
18521911
prompt: undefined,
18531912
};
18541913

1914+
static readonly storageAccessKey: CommandFlag = {
1915+
constName: 'storageAccessKey',
1916+
name: 'storage-access-key',
1917+
definition: {
1918+
defaultValue: '',
1919+
describe: 'storage access key for mirror node importer',
1920+
type: 'string',
1921+
dataMask: constants.STANDARD_DATAMASK,
1922+
},
1923+
prompt: undefined,
1924+
};
1925+
1926+
static readonly storageSecrets: CommandFlag = {
1927+
constName: 'storageSecrets',
1928+
name: 'storage-secrets',
1929+
definition: {
1930+
defaultValue: '',
1931+
describe: 'storage secret key for mirror node importer',
1932+
type: 'string',
1933+
dataMask: constants.STANDARD_DATAMASK,
1934+
},
1935+
prompt: undefined,
1936+
};
1937+
1938+
static readonly storageEndpoint: CommandFlag = {
1939+
constName: 'storageEndpoint',
1940+
name: 'storage-endpoint',
1941+
definition: {
1942+
defaultValue: '',
1943+
describe: 'storage endpoint URL for mirror node importer',
1944+
type: 'string',
1945+
dataMask: constants.STANDARD_DATAMASK,
1946+
},
1947+
prompt: undefined,
1948+
};
1949+
1950+
static readonly storageBucket: CommandFlag = {
1951+
constName: 'storageBucket',
1952+
name: 'storage-bucket',
1953+
definition: {
1954+
defaultValue: '',
1955+
describe: 'name of storage bucket for mirror node importer',
1956+
type: 'string',
1957+
dataMask: constants.STANDARD_DATAMASK,
1958+
},
1959+
prompt: undefined,
1960+
};
1961+
1962+
static readonly storageBucketPrefix: CommandFlag = {
1963+
constName: 'storageBucketPrefix',
1964+
name: 'storage-bucket-prefix',
1965+
definition: {
1966+
defaultValue: '',
1967+
describe: 'path prefix of storage bucket mirror node importer',
1968+
type: 'string',
1969+
},
1970+
prompt: undefined,
1971+
};
1972+
18551973
static readonly loadBalancerEnabled: CommandFlag = {
18561974
constName: 'loadBalancerEnabled',
18571975
name: 'load-balancer',
@@ -1945,6 +2063,16 @@ export class Flags {
19452063
Flags.stakeAmounts,
19462064
Flags.stateFile,
19472065
Flags.storageType,
2066+
Flags.gcsAccessKey,
2067+
Flags.gcsSecrets,
2068+
Flags.gcsEndpoint,
2069+
Flags.gcsBucket,
2070+
Flags.gcsBucketPrefix,
2071+
Flags.awsAccessKey,
2072+
Flags.awsSecrets,
2073+
Flags.awsEndpoint,
2074+
Flags.awsBucket,
2075+
Flags.awsBucketPrefix,
19482076
Flags.storageAccessKey,
19492077
Flags.storageSecrets,
19502078
Flags.storageEndpoint,

‎src/commands/mirror_node.ts

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -131,14 +131,18 @@ export class MirrorNodeCommand extends BaseCommand {
131131
}
132132

133133
let storageType = '';
134-
if (config.storageType && config.storageAccessKey && config.storageSecrets && config.storageEndpoint) {
134+
if (
135+
config.storageType !== constants.StorageType.MINIO_ONLY &&
136+
config.storageAccessKey &&
137+
config.storageSecrets &&
138+
config.storageEndpoint
139+
) {
135140
if (
136141
config.storageType === constants.StorageType.GCS_ONLY ||
137-
config.storageType === constants.StorageType.S3_AND_GCS ||
138-
config.storageType === constants.StorageType.GCS_AND_MINIO
142+
config.storageType === constants.StorageType.AWS_AND_GCS
139143
) {
140144
storageType = 'gcp';
141-
} else if (config.storageType === constants.StorageType.S3_ONLY) {
145+
} else if (config.storageType === constants.StorageType.AWS_ONLY) {
142146
storageType = 's3';
143147
} else {
144148
throw new IllegalArgumentError(`Invalid cloud storage type: ${config.storageType}`);

0 commit comments

Comments
 (0)