Add script to upload cloud gcp jars to appropriate bucket. (#203)

david-zlai · web-flow · commit 4bb48de8cee9 · 2025-01-14T16:28:58.000-05:00
## Summary ^^^ ``` (dev_chronon) davidhan@Davids-MacBook-Pro: ~/zipline/chronon (davidhan/add_upload_script) $ bash distribution/build_and_upload_gcp_artifacts.sh canary Working in /Users/davidhan/zipline/chronon Building wheel Processing ./api/py Preparing metadata (setup.py) ... done Collecting click (from zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/click-8.1.8-py3-none-any.whl Collecting thrift==0.21.0 (from zipline-ai==0.1.0.dev0) Using cached thrift-0.21.0-cp311-cp311-macosx_15_0_arm64.whl File was already downloaded /Users/davidhan/zipline/chronon/thrift-0.21.0-cp311-cp311-macosx_15_0_arm64.whl Collecting google-cloud-storage==2.19.0 (from zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/google_cloud_storage-2.19.0-py2.py3-none-any.whl Collecting google-auth<3.0dev,>=2.26.1 (from google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/google_auth-2.37.0-py2.py3-none-any.whl Collecting google-api-core<3.0.0dev,>=2.15.0 (from google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/google_api_core-2.24.0-py3-none-any.whl Collecting google-cloud-core<3.0dev,>=2.3.0 (from google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/google_cloud_core-2.4.1-py2.py3-none-any.whl Collecting google-resumable-media>=2.7.2 (from google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/google_resumable_media-2.7.2-py2.py3-none-any.whl Collecting requests<3.0.0dev,>=2.18.0 (from google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/requests-2.32.3-py3-none-any.whl Collecting google-crc32c<2.0dev,>=1.0 (from google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/google_crc32c-1.6.0-cp311-cp311-macosx_12_0_arm64.whl Collecting six>=1.7.2 (from thrift==0.21.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/six-1.17.0-py2.py3-none-any.whl Collecting googleapis-common-protos<2.0.dev0,>=1.56.2 (from google-api-core<3.0.0dev,>=2.15.0->google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/googleapis_common_protos-1.66.0-py2.py3-none-any.whl Collecting protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0.dev0,>=3.19.5 (from google-api-core<3.0.0dev,>=2.15.0->google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/protobuf-5.29.3-cp38-abi3-macosx_10_9_universal2.whl Collecting proto-plus<2.0.0dev,>=1.22.3 (from google-api-core<3.0.0dev,>=2.15.0->google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/proto_plus-1.25.0-py3-none-any.whl Collecting cachetools<6.0,>=2.0.0 (from google-auth<3.0dev,>=2.26.1->google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/cachetools-5.5.0-py3-none-any.whl Collecting pyasn1-modules>=0.2.1 (from google-auth<3.0dev,>=2.26.1->google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/pyasn1_modules-0.4.1-py3-none-any.whl Collecting rsa<5,>=3.1.4 (from google-auth<3.0dev,>=2.26.1->google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/rsa-4.9-py3-none-any.whl Collecting charset-normalizer<4,>=2 (from requests<3.0.0dev,>=2.18.0->google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/charset_normalizer-3.4.1-cp311-cp311-macosx_10_9_universal2.whl Collecting idna<4,>=2.5 (from requests<3.0.0dev,>=2.18.0->google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/idna-3.10-py3-none-any.whl Collecting urllib3<3,>=1.21.1 (from requests<3.0.0dev,>=2.18.0->google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/urllib3-2.3.0-py3-none-any.whl Collecting certifi>=2017.4.17 (from requests<3.0.0dev,>=2.18.0->google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/certifi-2024.12.14-py3-none-any.whl Collecting pyasn1<0.7.0,>=0.4.6 (from pyasn1-modules>=0.2.1->google-auth<3.0dev,>=2.26.1->google-cloud-storage==2.19.0->zipline-ai==0.1.0.dev0) File was already downloaded /Users/davidhan/zipline/chronon/pyasn1-0.6.1-py3-none-any.whl Building wheels for collected packages: zipline-ai Building wheel for zipline-ai (setup.py) ... done Created wheel for zipline-ai: filename=zipline_ai-0.1.0.dev0-py3-none-any.whl size=7316 sha256=5327d4bc284b4c02ac1f1cc0078a748a59f027f2bd545f4675362f061467fa6b Stored in directory: /private/var/folders/2p/h5v8s0515xv20cgprdjngttr0000gn/T/pip-ephem-wheel-cache-oqk_7n9r/wheels/82/21/73/c9719a19192e6cf69f01ea4fffb963c5d02e6228d8b875f326 Successfully built zipline-ai Building jars [info] welcome to sbt 1.8.2 (Amazon.com Inc. Java 11.0.25) [info] loading settings for project chronon-build from plugins.sbt ... [info] loading project definition from /Users/davidhan/zipline/chronon/project [info] loading settings for project root from build.sbt,version.sbt ... [info] resolving key references (17954 settings) ... [info] set current project to chronon (in build file:/Users/davidhan/zipline/chronon/) "Generating files from thrift files at: /Users/davidhan/zipline/chronon/api/thrift/ into folder /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main" + rm -rf /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main + mkdir -p /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main + thrift -version [out] Thrift version 0.13.0 Processing file: /Users/davidhan/zipline/chronon/api/thrift/orchestration.thrift + thrift --gen java:generated_annotations=suppress -out /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main /Users/davidhan/zipline/chronon/api/thrift/orchestration.thrift Processing file: /Users/davidhan/zipline/chronon/api/thrift/api.thrift + thrift --gen java:generated_annotations=suppress -out /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main /Users/davidhan/zipline/chronon/api/thrift/api.thrift Processing file: /Users/davidhan/zipline/chronon/api/thrift/observability.thrift + thrift --gen java:generated_annotations=suppress -out /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main /Users/davidhan/zipline/chronon/api/thrift/observability.thrift Processing file: /Users/davidhan/zipline/chronon/api/thrift/common.thrift + thrift --gen java:generated_annotations=suppress -out /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main /Users/davidhan/zipline/chronon/api/thrift/common.thrift Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/Cardinality.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/TileSeriesKey.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/DriftMetric.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/TileKey.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/TileSummarySeries.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/TileSummary.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/TileDrift.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/TileDriftSeries.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/DriftSpec.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/Artifact.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/ScanStrategy.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/LogicalNode.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/LogicalType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/ModelNodeType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/TableRange.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/TableDependency.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/TableNodeType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/PhysicalNodeKey.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/GroupByNodeType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/ArtifactRange.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/NodeConnections.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/Table.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/TabularData.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/NodeGraph.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/JoinNodeType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/KvEntry.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/NodeInfo.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/PhysicalNodeType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/NodeKey.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/PhysicalNodeInstance.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/PhysicalNode.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/StagingQueryNodeType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/KvRange.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/TabularDataType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/KvDependency.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/Dependency.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Join.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/GroupByServingInfo.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/ExternalPart.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/BootstrapPart.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/JoinPart.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Window.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/DataKind.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Accuracy.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Operation.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Derivation.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/StagingQuery.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/DataSpec.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/MetaData.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/EntitySource.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/LabelParts.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/JoinSource.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Query.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/TimeUnit.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/AggregationPart.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/DataField.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/ExternalSource.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Aggregation.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Source.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/EventSource.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/TDataType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/ModelType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/GroupBy.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Model.java [warn] multiple main classes detected: run 'show discoveredMainClasses' to see the list [info] Assembly jar up to date: /Users/davidhan/zipline/chronon/cloud_gcp/target/scala-2.12/cloud_gcp-assembly-0.1.0-SNAPSHOT.jar [success] Total time: 3 s, completed Jan 14, 2025, 12:50:56 PM [info] welcome to sbt 1.8.2 (Amazon.com Inc. Java 11.0.25) [info] loading settings for project chronon-build from plugins.sbt ... [info] loading project definition from /Users/davidhan/zipline/chronon/project [info] loading settings for project root from build.sbt,version.sbt ... [info] resolving key references (17954 settings) ... [info] set current project to chronon (in build file:/Users/davidhan/zipline/chronon/) "Generating files from thrift files at: /Users/davidhan/zipline/chronon/api/thrift/ into folder /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main" + rm -rf /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main + mkdir -p /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main + thrift -version [out] Thrift version 0.13.0 Processing file: /Users/davidhan/zipline/chronon/api/thrift/orchestration.thrift + thrift --gen java:generated_annotations=suppress -out /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main /Users/davidhan/zipline/chronon/api/thrift/orchestration.thrift Processing file: /Users/davidhan/zipline/chronon/api/thrift/api.thrift + thrift --gen java:generated_annotations=suppress -out /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main /Users/davidhan/zipline/chronon/api/thrift/api.thrift Processing file: /Users/davidhan/zipline/chronon/api/thrift/observability.thrift + thrift --gen java:generated_annotations=suppress -out /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main /Users/davidhan/zipline/chronon/api/thrift/observability.thrift Processing file: /Users/davidhan/zipline/chronon/api/thrift/common.thrift + thrift --gen java:generated_annotations=suppress -out /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main /Users/davidhan/zipline/chronon/api/thrift/common.thrift Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/Cardinality.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/TileSeriesKey.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/DriftMetric.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/TileKey.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/TileSummarySeries.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/TileSummary.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/TileDrift.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/TileDriftSeries.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/observability/DriftSpec.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/Artifact.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/ScanStrategy.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/LogicalNode.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/LogicalType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/ModelNodeType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/TableRange.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/TableDependency.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/TableNodeType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/PhysicalNodeKey.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/GroupByNodeType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/ArtifactRange.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/NodeConnections.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/Table.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/TabularData.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/NodeGraph.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/JoinNodeType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/KvEntry.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/NodeInfo.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/PhysicalNodeType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/NodeKey.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/PhysicalNodeInstance.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/PhysicalNode.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/StagingQueryNodeType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/KvRange.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/TabularDataType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/KvDependency.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/orchestration/Dependency.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Join.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/GroupByServingInfo.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/ExternalPart.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/BootstrapPart.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/JoinPart.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Window.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/DataKind.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Accuracy.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Operation.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Derivation.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/StagingQuery.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/DataSpec.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/MetaData.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/EntitySource.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/LabelParts.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/JoinSource.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Query.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/TimeUnit.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/AggregationPart.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/DataField.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/ExternalSource.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Aggregation.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Source.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/EventSource.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/TDataType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/ModelType.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/GroupBy.java Processing file: /Users/davidhan/zipline/chronon/api/target/scala-2.12/src_managed/main/ai/chronon/api/Model.java [info] Assembly jar up to date: /Users/davidhan/zipline/chronon/cloud_gcp_submitter/target/scala-2.12/cloud_gcp_submitter-assembly-0.1.0-SNAPSHOT.jar [success] Total time: 3 s, completed Jan 14, 2025, 12:51:08 PM Are you sure you want to upload to these customer ids: canary 1) Yes 2) No #? 1 + for element in '"${customer_ids_to_upload[@]}"' + ELEMENT_JAR_PATH=gs://zipline-artifacts-canary/jars + gcloud storage cp /Users/davidhan/zipline/chronon/cloud_gcp/target/scala-2.12/cloud_gcp-assembly-0.1.0-SNAPSHOT.jar gs://zipline-artifacts-canary/jars WARNING: Parallel composite upload was turned ON to get the best performance on uploading large objects. If you would like to opt-out and instead perform a normal upload, run: `gcloud config set storage/parallel_composite_upload_enabled False` If you would like to disable this warning, run: `gcloud config set storage/parallel_composite_upload_enabled True` Note that with parallel composite uploads, your object might be uploaded as a composite object (https://cloud.google.com/storage/docs/composite-objects), which means that any user who downloads your object will need to use crc32c checksums to verify data integrity. gcloud storage is capable of computing crc32c checksums, but this might pose a problem for other clients. Copying file:///Users/davidhan/zipline/chronon/cloud_gcp/target/scala-2.12/cloud_gcp-assembly-0.1.0-SNAPSHOT.jar to gs://zipline-artifacts-canary/jars/cloud_gcp-assembly-0.1.0-SNAPSHOT.jar Completed files 4/1 | 155.8MiB/155.8MiB | 9.8MiB/s Average throughput: 11.2MiB/s + gcloud storage cp /Users/davidhan/zipline/chronon/cloud_gcp_submitter/target/scala-2.12/cloud_gcp_submitter-assembly-0.1.0-SNAPSHOT.jar gs://zipline-artifacts-canary/jars WARNING: Parallel composite upload was turned ON to get the best performance on uploading large objects. If you would like to opt-out and instead perform a normal upload, run: `gcloud config set storage/parallel_composite_upload_enabled False` If you would like to disable this warning, run: `gcloud config set storage/parallel_composite_upload_enabled True` Note that with parallel composite uploads, your object might be uploaded as a composite object (https://cloud.google.com/storage/docs/composite-objects), which means that any user who downloads your object will need to use crc32c checksums to verify data integrity. gcloud storage is capable of computing crc32c checksums, but this might pose a problem for other clients. Copying file:///Users/davidhan/zipline/chronon/cloud_gcp_submitter/target/scala-2.12/cloud_gcp_submitter-assembly-0.1.0-SNAPSHOT.jar to gs://zipline-artifacts-canary/jars/cloud_gcp_submitter-assembly-0.1.0-SNAPSHOT.jar Completed files 4/1 | 155.8MiB/155.8MiB | 13.0MiB/s Average throughput: 9.9MiB/s + gcloud storage cp zipline_ai-0.1.0.dev0-py3-none-any.whl gs://zipline-artifacts-canary/jars Copying file://zipline_ai-0.1.0.dev0-py3-none-any.whl to gs://zipline-artifacts-canary/jars/zipline_ai-0.1.0.dev0-py3-none-any.whl Completed files 1/1 | 7.1kiB/7.1kiB + echo Succeeded Succeeded + break ``` ## Checklist - [ ] Added Unit Tests - [ ] Covered by existing CI - [ ] Integration tested - [ ] Documentation update  ## Summary by CodeRabbit ## Summary by CodeRabbit - **New Features** - Added a new Bash script to automate the building and uploading of Google Cloud Platform (GCP) artifacts. - Supports automated JAR file generation and cloud storage upload for specific projects. - Includes error handling for missing arguments and files. - Updated Python version declarations to support multiple versions. - **Chores** - Introduced build and deployment automation for GCP-related components.  --- - To see the specific tasks where the Asana app for GitHub is being used, see below: - https://app.asana.com/0/0/1209143482009686
diff --git a/.tool-versions b/.tool-versions
@@ -2,4 +2,6 @@ java corretto-11.0.25.9.1
 scala 2.12.20
 asdf-plugin-manager 1.4.0
 sbt 1.8.2
-python 3.7.17
+python
+    3.7.17
+    3.11.0
diff --git a/distribution/build_and_upload_gcp_artifacts.sh b/distribution/build_and_upload_gcp_artifacts.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+
+SCRIPT_DIRECTORY=$(dirname -- "$(realpath -- "$0")")
+CHRONON_ROOT_DIR=$(dirname "$SCRIPT_DIRECTORY")
+
+echo "Working in $CHRONON_ROOT_DIR"
+cd $CHRONON_ROOT_DIR
+
+echo "Building wheel"
+#Check python version >= 3.9
+MAJOR_PYTHON_VERSION=$(python --version | cut -d " " -f2 | cut -d "." -f 1)
+MINOR_PYTHON_VERSION=$(python --version | cut -d " " -f2 | cut -d "." -f 2)
+
+EXPECTED_MINIMUM_MAJOR_PYTHON_VERSION=3
+EXPECTED_MINIMUM_MINOR_PYTHON_VERSION=9
+
+if [[ $EXPECTED_MINIMUM_MAJOR_PYTHON_VERSION -gt $MAJOR_PYTHON_VERSION ]] ; then
+    echo "Failed major version of $MAJOR_PYTHON_VERSION. Expecting python version of at least $EXPECTED_MINIMUM_MAJOR_PYTHON_VERSION.$EXPECTED_MINIMUM_MINOR_PYTHON_VERSION to build wheel. Your version is $(python --version)"
+    exit 1
+fi
+
+if [[ EXPECTED_MINIMUM_MINOR_PYTHON_VERSION -gt MINOR_PYTHON_VERSION ]] ; then
+    echo "Failed minor version of $MINOR_PYTHON_VERSION. Expecting python version of at least $EXPECTED_MINIMUM_MAJOR_PYTHON_VERSION.$EXPECTED_MINIMUM_MINOR_PYTHON_VERSION to build wheel. Your version is $(python --version)"
+    exit 1
+fi
+
+thrift --gen py -out api/py/ai/chronon api/thrift/common.thrift
+thrift --gen py -out api/py/ai/chronon api/thrift/api.thrift
+VERSION=$(cat version.sbt | cut -d " " -f3 | tr -d '"') pip wheel api/py
+EXPECTED_ZIPLINE_WHEEL="zipline_ai-0.1.0.dev0-py3-none-any.whl"
+if [ ! -f "$EXPECTED_ZIPLINE_WHEEL" ]; then
+    echo "$EXPECTED_ZIPLINE_WHEEL not found"
+    exit 1
+fi
+
+echo "Building jars"
+sbt cloud_gcp/assembly
+sbt cloud_gcp_submitter/assembly
+
+CLOUD_GCP_JAR="$CHRONON_ROOT_DIR/cloud_gcp/target/scala-2.12/cloud_gcp-assembly-0.1.0-SNAPSHOT.jar"
+CLOUD_GCP_SUBMITTER_JAR="$CHRONON_ROOT_DIR/cloud_gcp_submitter/target/scala-2.12/cloud_gcp_submitter-assembly-0.1.0-SNAPSHOT.jar"
+
+if [ ! -f "$CLOUD_GCP_JAR" ]; then
+    echo "$CLOUD_GCP_JAR not found"
+    exit 1
+fi
+
+if [ ! -f "$CLOUD_GCP_SUBMITTER_JAR" ]; then
+    echo "$CLOUD_GCP_SUBMITTER_JAR not found"
+    exit 1
+fi
+
+# all customer ids
+ALL_CUSTOMER_IDS=("canary" "etsy")
+
+# Takes in array of customer ids
+function upload_to_gcp() {
+  customer_ids_to_upload=("$@")
+  echo "Are you sure you want to upload to these customer ids: ${customer_ids_to_upload[*]}"
+  select yn in "Yes" "No"; do
+      case $yn in
+          Yes )
+              set -euxo pipefail
+              for element in "${customer_ids_to_upload[@]}"
+              do
+                ELEMENT_JAR_PATH=gs://zipline-artifacts-$element/jars
+                gcloud storage cp "$CLOUD_GCP_JAR" "$ELEMENT_JAR_PATH";
+                gcloud storage cp "$CLOUD_GCP_SUBMITTER_JAR" "$ELEMENT_JAR_PATH";
+                gcloud storage cp "$EXPECTED_ZIPLINE_WHEEL" "$ELEMENT_JAR_PATH"
+              done
+              echo "Succeeded"
+              break;;
+          No ) exit 0;;
+      esac
+  done
+}
+
+# check if $1 (single customer id mode) has been set
+if [ -z "$1" ]; then
+  upload_to_gcp "${ALL_CUSTOMER_IDS[@]}"
+else
+  upload_to_gcp "$1"
+fi
