# Lint rules to ignore LINTIGNORESINGLEFIGHT='internal/sync/singleflight/singleflight.go:.+error should be the last type' LINT_IGNORE_S3MANAGER_INPUT='feature/s3/manager/upload.go:.+struct field SSEKMSKeyId should be SSEKMSKeyID' # Names of these are tied to endpoint rules and they're internal so ignore them LINT_IGNORE_AWSRULESFN_ARN='internal/endpoints/awsrulesfn/arn.go' LINT_IGNORE_AWSRULESFN_PARTITION='internal/endpoints/awsrulesfn/partition.go' LINT_IGNORE_PRIVATE_METRICS='aws/middleware/private/metrics' UNIT_TEST_TAGS= BUILD_TAGS=-tags "example,codegen,integration,ec2env,perftest" SNAPSHOT_TAGS=-tags "snapshot" SMITHY_GO_SRC ?= $(abspath $(shell pwd)/..)/smithy-go SDK_MIN_GO_VERSION ?= 1.24 EACHMODULE_FAILFAST ?= true EACHMODULE_FAILFAST_FLAG=-fail-fast=${EACHMODULE_FAILFAST} EACHMODULE_CONCURRENCY ?= 1 EACHMODULE_CONCURRENCY_FLAG=-c ${EACHMODULE_CONCURRENCY} EACHMODULE_SKIP ?= EACHMODULE_SKIP_FLAG=-skip="${EACHMODULE_SKIP}" EACHMODULE_FLAGS=${EACHMODULE_CONCURRENCY_FLAG} ${EACHMODULE_FAILFAST_FLAG} ${EACHMODULE_SKIP_FLAG} RUN_NONE=-run NONE RUN_INTEG=-run '^TestInteg_' CODEGEN_RESOURCES_PATH=$(shell pwd)/codegen/smithy-aws-go-codegen/src/main/resources/software/amazon/smithy/aws/go/codegen CODEGEN_API_MODELS_PATH=$(shell pwd)/codegen/sdk-codegen/aws-models ENDPOINTS_JSON=${CODEGEN_RESOURCES_PATH}/endpoints.json ENDPOINT_PREFIX_JSON=${CODEGEN_RESOURCES_PATH}/endpoint-prefix.json LICENSE_FILE=$(shell pwd)/LICENSE.txt SMITHY_GO_VERSION ?= PRE_RELEASE_VERSION ?= RELEASE_MANIFEST_FILE ?= RELEASE_CHGLOG_DESC_FILE ?= REPOTOOLS_VERSION ?= latest REPOTOOLS_MODULE = github.com/awslabs/aws-go-multi-module-repository-tools REPOTOOLS_CMD_ANNOTATE_STABLE_GEN = ${REPOTOOLS_MODULE}/cmd/annotatestablegen@${REPOTOOLS_VERSION} REPOTOOLS_CMD_MAKE_RELATIVE = ${REPOTOOLS_MODULE}/cmd/makerelative@${REPOTOOLS_VERSION} REPOTOOLS_CMD_CALCULATE_RELEASE = ${REPOTOOLS_MODULE}/cmd/calculaterelease@${REPOTOOLS_VERSION} REPOTOOLS_CMD_UPDATE_REQUIRES = ${REPOTOOLS_MODULE}/cmd/updaterequires@${REPOTOOLS_VERSION} REPOTOOLS_CMD_UPDATE_MODULE_METADATA = ${REPOTOOLS_MODULE}/cmd/updatemodulemeta@${REPOTOOLS_VERSION} REPOTOOLS_CMD_GENERATE_CHANGELOG = ${REPOTOOLS_MODULE}/cmd/generatechangelog@${REPOTOOLS_VERSION} REPOTOOLS_CMD_CHANGELOG = ${REPOTOOLS_MODULE}/cmd/changelog@${REPOTOOLS_VERSION} REPOTOOLS_CMD_TAG_RELEASE = ${REPOTOOLS_MODULE}/cmd/tagrelease@${REPOTOOLS_VERSION} REPOTOOLS_CMD_EDIT_MODULE_DEPENDENCY = ${REPOTOOLS_MODULE}/cmd/editmoduledependency@${REPOTOOLS_VERSION} REPOTOOLS_CALCULATE_RELEASE_VERBOSE ?= false REPOTOOLS_CALCULATE_RELEASE_VERBOSE_FLAG=-v=${REPOTOOLS_CALCULATE_RELEASE_VERBOSE} REPOTOOLS_CALCULATE_RELEASE_ADDITIONAL_ARGS ?= DEV_SERVICE ?= s3 ifneq ($(PRE_RELEASE_VERSION),) REPOTOOLS_CALCULATE_RELEASE_ADDITIONAL_ARGS += -preview=${PRE_RELEASE_VERSION} endif .PHONY: all all: generate unit ################### # Code Generation # ################### .PHONY: generate smithy-generate smithy-generate-protocol-tests smithy-build-% smithy-clean smithy-go-publish-local format \ gen-config-asserts gen-repo-mod-replace gen-mod-replace-smithy gen-mod-dropreplace-smithy-% gen-aws-ptrs tidy-modules-% \ add-module-license-files sync-models sync-endpoints-model sync-endpoints.json clone-v1-models gen-internal-codegen \ sync-api-models copy-attributevalue-feature min-go-version-% update-requires smithy-annotate-stable \ update-module-metadata download-modules-% generate: smithy-generate smithy-generate-protocol-tests update-requires gen-repo-mod-replace update-module-metadata smithy-annotate-stable \ gen-config-asserts gen-internal-codegen copy-attributevalue-feature gen-mod-dropreplace-smithy-. min-go-version-. \ tidy-modules-. test-update-snapshot-internal_protocoltest add-module-license-files gen-aws-ptrs format generate-tmpreplace-smithy: smithy-generate smithy-generate-protocol-tests update-requires gen-repo-mod-replace gen-mod-replace-smithy-. update-module-metadata smithy-annotate-stable \ gen-config-asserts gen-internal-codegen copy-attributevalue-feature min-go-version-. \ tidy-modules-. add-module-license-files gen-aws-ptrs format gen-mod-dropreplace-smithy-. reset-sum # stripped-down regenerate script that eliminates a lot of the cruft you don't # need in development (that takes time) # set DEV_SERVICE to the service package name (e.g. ec2, bedrockruntime) # # e.g.: # SMITHY_GO_BUILD_API=com.amazonaws.sqs DEV_SERVICE=sqs make generate-dev generate-dev: smithy-generate update-requires gen-repo-mod-replace gen-mod-replace-smithy-config gen-mod-replace-smithy-aws gen-mod-replace-smithy-service_${DEV_SERVICE} update-module-metadata smithy-annotate-stable \ gen-config-asserts gen-internal-codegen tidy-modules-config tidy-modules-aws tidy-modules-service_${DEV_SERVICE} format-dev generate-protocoltest-dev: smithy-generate smithy-generate-protocol-tests update-requires gen-repo-mod-replace gen-mod-replace-smithy-config gen-mod-replace-smithy-aws gen-mod-replace-smithy-internal_protocoltest_${DEV_SERVICE} update-module-metadata smithy-annotate-stable \ gen-config-asserts gen-internal-codegen tidy-modules-config tidy-modules-aws tidy-modules-internal_protocoltest_${DEV_SERVICE} format-protocoltest-dev reset-sum: find . -name go.sum -exec git checkout -- {} \; # Codegen is split into two loops: # 1. smithy-generate: builds SDK services and kitchensink tests (protocol-test-codegen is excluded) # 2. smithy-generate-protocol-tests: builds protocol tests independently (generates smithy-build.json, runs buildSdk, copies output) # Both are invoked by the top-level `generate` target. To regenerate only one, # call the specific target directly. smithy-generate: cd codegen && ./gradlew clean build -Plog-tests && ./gradlew clean smithy-generate-protocol-tests: cd codegen && SMITHY_GO_BUILD_API= ./gradlew :protocol-test-codegen:build -Plog-tests && ./gradlew :protocol-test-codegen:clean # suffix-to-path pattern # Targets with pattern suffix -% convert the suffix to a path by: # 1. Stripping the target prefix (e.g., "go-build-modules-", "build-modules-", "ci-lint-") # 2. Converting underscores to slashes (e.g., internal_protocoltest -> internal/protocoltest) # The resulting path is passed to the eachmodule cmd via the -p flag to specify which # modules to operate on. Using "." targets all modules in the current directory. # Usage examples: # make go-build-modules-internal_protocoltest # make ci-lint-internal_protocoltest smithy-build-%: @# See suffix-to-path pattern. Defines the smithy API model service id starts with. @# e.g. smithy-build-com.amazonaws.rds @# e.g. smithy-build-com.amazonaws.rds#AmazonRDSv19 cd codegen && \ SMITHY_GO_BUILD_API="$(subst smithy-build-,,$@)" ./gradlew clean build -Plog-tests smithy-annotate-stable: go run ${REPOTOOLS_CMD_ANNOTATE_STABLE_GEN} smithy-clean: cd codegen && ./gradlew clean smithy-go-publish-local: rm -rf /tmp/smithy-go-local git clone https://github.com/aws/smithy-go /tmp/smithy-go-local make -C /tmp/smithy-go-local smithy-clean smithy-publish-local format: gofmt -w -s . format-dev: gofmt -w -s service/${DEV_SERVICE} format-protocoltest-dev: gofmt -w -s internal/protocoltest/${DEV_SERVICE} gen-config-asserts: @echo "Generating SDK config package implementor assertions" cd config \ && go mod tidy \ && go generate gen-internal-codegen: @echo "Generating internal/codegen" cd internal/codegen \ && go mod tidy \ && go generate gen-repo-mod-replace: @echo "Generating go.mod replace for repo modules" go run ${REPOTOOLS_CMD_MAKE_RELATIVE} gen-mod-replace-smithy-%: @# See suffix-to-path pattern. Defines build filter for modules to add `-replace` to @# @# SMITHY_GO_SRC environment variable is the path to add replace to @# @# e.g. gen-mod-replace-smithy-service_ssooidc cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst gen-mod-replace-smithy-,,$@)) ${EACHMODULE_FLAGS} \ "go mod edit -replace github.com/aws/smithy-go=${SMITHY_GO_SRC}" gen-mod-dropreplace-smithy-%: @# See suffix-to-path pattern. Defines build filter for modules to add `-dropreplace` to @# @# e.g. gen-mod-dropreplace-smithy-service_ssooidc cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst gen-mod-dropreplace-smithy-,,$@)) ${EACHMODULE_FLAGS} \ "go mod edit -dropreplace github.com/aws/smithy-go" gen-aws-ptrs: cd aws && go generate tidy-modules-%: @# See suffix-to-path pattern. Runs `go mod tidy` cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst tidy-modules-,,$@)) ${EACHMODULE_FLAGS} \ "go mod tidy" download-modules-%: @# See suffix-to-path pattern. Runs `go mod download all` cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst download-modules-,,$@)) ${EACHMODULE_FLAGS} \ "go mod download all" add-module-license-files: cd internal/repotools/cmd/eachmodule && \ go run . -skip-root \ "cp $(LICENSE_FILE) ." sync-models: sync-endpoints-model sync-api-models sync-endpoints-model: sync-endpoints.json sync-endpoints.json: [[ ! -z "${ENDPOINTS_MODEL}" ]] && cp ${ENDPOINTS_MODEL} ${ENDPOINTS_JSON} || echo "ENDPOINTS_MODEL not set, must not be empty" clone-v1-models: rm -rf /tmp/aws-sdk-go-model-sync git clone https://github.com/aws/aws-sdk-go.git --depth 1 /tmp/aws-sdk-go-model-sync sync-api-models: cd internal/repotools/cmd/syncAPIModels && \ go run . \ -m ${API_MODELS} \ -o ${CODEGEN_API_MODELS_PATH} copy-attributevalue-feature: cd ./feature/dynamodbstreams/attributevalue && \ find . -name "*.go" | grep -v "doc.go" | grep -v "go_module_metadata.go" | xargs -I % rm % && \ find ../../dynamodb/attributevalue -name "*.go" | grep -v "doc.go" | grep -v "go_module_metadata.go" | xargs -I % cp % . && \ ls *.go | grep -v "convert.go" | grep -v "doc.go" | grep -v "go_module_metadata.go" | \ xargs -I % sed -i.bk -E 's:github.com/aws/aws-sdk-go-v2/(service|feature)/dynamodb:github.com/aws/aws-sdk-go-v2/\1/dynamodbstreams:g' % && \ ls *.go | grep -v "convert.go" | grep -v "doc.go" | grep -v "go_module_metadata.go" | \ xargs -I % sed -i.bk 's:DynamoDB:DynamoDBStreams:g' % && \ ls *.go | grep -v "doc.go" | grep -v "go_module_metadata.go" | \ xargs -I % sed -i.bk 's:dynamodb\.:dynamodbstreams.:g' % && \ sed -i.bk 's:streams\.:ddbtypes.:g' "convert.go" && \ sed -i.bk 's:ddb\.:streams.:g' "convert.go" && \ sed -i.bk 's:ddbtypes\.:ddb.:g' "convert.go" &&\ sed -i.bk 's:Streams::g' "convert.go" && \ rm -rf ./*.bk && \ go mod tidy && \ gofmt -w -s . && \ go test . min-go-version-%: cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst min-go-version-,,$@)) ${EACHMODULE_FLAGS} \ "go mod edit -go=${SDK_MIN_GO_VERSION}" update-requires: go run ${REPOTOOLS_CMD_UPDATE_REQUIRES} update-module-metadata: go run ${REPOTOOLS_CMD_UPDATE_MODULE_METADATA} ################ # Unit Testing # ################ .PHONY: unit unit-race unit-test unit-race-test unit-race-modules-% unit-modules-% build build-modules-% \ go-build-modules-% test test-race-modules-% test-modules-% cachedep cachedep-modules-% api-diff-modules-% unit: lint unit-modules-. unit-race: lint unit-race-modules-. unit-test: test-modules-. unit-race-test: test-race-modules-. unit-race-modules-%: @# See suffix-to-path pattern. Runs `go test` with `-race` flag cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst unit-race-modules-,,$@)) ${EACHMODULE_FLAGS} \ "go test -timeout=2m ${UNIT_TEST_TAGS} -race -cpu=4 ./..." unit-modules-%: @# See suffix-to-path pattern. Runs `go test` cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst unit-modules-,,$@)) ${EACHMODULE_FLAGS} \ "go test -timeout=2m ${UNIT_TEST_TAGS} ./..." build: build-modules-. build-modules-%: @# See suffix-to-path pattern. Builds all modules and test files via `go test -run NONE` cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst build-modules-,,$@)) ${EACHMODULE_FLAGS} \ "go test ${BUILD_TAGS} ${RUN_NONE} ./..." TEMP_FILE := $(shell mktemp) build-tagged-modules: @# Compiles modules tagged with BUILD_TAGS. @# This just ensures the modules compile correctly and doesn't actually produce any executable. @# This also runs "vet" analyzers on the module cd ./internal/repotools/cmd/findtaggedmodules \ && go run . $(BUILD_TAGS) > "$(TEMP_FILE)"; cd ./internal/repotools/cmd/eachmodule \ && while read module; do \ echo "Testing module: $$module"; \ (go run . -p "$$module" ${EACHMODULE_FLAGS} "go test ${BUILD_TAGS} ${RUN_NONE} -vet=all ./...") || { echo "Tests failed for module: $$module"; exit 1; }; \ done < "$(TEMP_FILE)"; go-build-modules-%: @# See suffix-to-path pattern. Builds all modules without test files cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst go-build-modules-,,$@)) ${EACHMODULE_FLAGS} \ "go build ${BUILD_TAGS} ./..." test: unit-modules-. test-race-vet-modules-%: @# See suffix-to-path pattern. Runs `go test` with `-race -vet` flags cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst test-race-vet-modules-,,$@)) ${EACHMODULE_FLAGS} \ "go test -timeout=2m ${UNIT_TEST_TAGS} -race -vet=all -cpu=4 ./..." test-check-snapshot-%: cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst test-check-snapshot-,,$@)) ${EACHMODULE_FLAGS} \ "go test ${SNAPSHOT_TAGS} -run TestCheckSnapshot ./..." test-update-snapshot-%: cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst test-update-snapshot-,,$@)) ${EACHMODULE_FLAGS} \ "go test ${SNAPSHOT_TAGS} -run TestUpdateSnapshot ./..." test-ci-check-snapshot-%: cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst test-ci-check-snapshot-,,$@)) ${EACHMODULE_FLAGS} \ "go test ${SNAPSHOT_TAGS} -run TestCheckSnapshot -failfast ./..." cachedep: cachedep-modules-. cachedep-modules-%: @# See suffix-to-path pattern. Runs `go mod download` cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst cachedep-modules-,,$@)) ${EACHMODULE_FLAGS} \ "go mod download" api-diff-modules-%: @# See suffix-to-path pattern. Runs `gorelease` @# @# Requires golang.org/x/exp/cmd/gorelease to be available in the GOPATH. cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst api-diff-modules-,,$@)) \ -fail-fast=true \ -c 1 \ -skip="internal/repotools" \ "$$(go env GOPATH)/bin/gorelease" ############## # CI Testing # ############## .PHONY: ci-test-no-generate ci-test-no-generate-no-race ci-test-no-generate: lint build-tagged-modules test-race-vet-modules-. ci-test-no-generate-no-race: lint build-tagged-modules unit-modules-. ####################### # Integration Testing # ####################### .PHONY: integration integ-modules-% cleanup-integ-buckets integration: integ-modules-service integ-modules-feature integ-modules-%: @# See suffix-to-path pattern. Runs `go test` on modules with the `integration` tag cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst integ-modules-,,$@)) ${EACHMODULE_FLAGS} \ "go test -timeout=10m -tags "integration" -v ${RUN_INTEG} -count 1 ./..." cleanup-integ-buckets: @echo "Cleaning up SDK integration resources" go run -tags "integration" ./internal/awstesting/cmd/bucket_cleanup/main.go "aws-sdk-go-integration" ############## # Benchmarks # ############## .PHONY: bench bench-modules-% bench: bench-modules-. bench-modules-%: @# See suffix-to-path pattern. Runs benchmark tests cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst bench-modules-,,$@)) ${EACHMODULE_FLAGS} \ "go test -timeout=10m -bench . --benchmem ${BUILD_TAGS} ${RUN_NONE} ./..." ##################### # Release Process # ##################### .PHONY: preview-release pre-release-validation release ls-changes: go run ${REPOTOOLS_CMD_CHANGELOG} ls preview-release: go run ${REPOTOOLS_CMD_CALCULATE_RELEASE} ${REPOTOOLS_CALCULATE_RELEASE_VERBOSE_FLAG} ${REPOTOOLS_CALCULATE_RELEASE_ADDITIONAL_ARGS} pre-release-validation: @if [[ -z "${RELEASE_MANIFEST_FILE}" ]]; then \ echo "RELEASE_MANIFEST_FILE is required to specify the file to write the release manifest" && false; \ fi @if [[ -z "${RELEASE_CHGLOG_DESC_FILE}" ]]; then \ echo "RELEASE_CHGLOG_DESC_FILE is required to specify the file to write the release notes" && false; \ fi release: pre-release-validation go run ${REPOTOOLS_CMD_CALCULATE_RELEASE} -o ${RELEASE_MANIFEST_FILE} ${REPOTOOLS_CALCULATE_RELEASE_VERBOSE_FLAG} ${REPOTOOLS_CALCULATE_RELEASE_ADDITIONAL_ARGS} go run ${REPOTOOLS_CMD_UPDATE_REQUIRES} -release ${RELEASE_MANIFEST_FILE} go run ${REPOTOOLS_CMD_UPDATE_MODULE_METADATA} -release ${RELEASE_MANIFEST_FILE} go run ${REPOTOOLS_CMD_GENERATE_CHANGELOG} -release ${RELEASE_MANIFEST_FILE} -o ${RELEASE_CHGLOG_DESC_FILE} go run ${REPOTOOLS_CMD_CHANGELOG} rm -all go run ${REPOTOOLS_CMD_TAG_RELEASE} -release ${RELEASE_MANIFEST_FILE} ############## # Repo Tools # ############## .PHONY: install-repotools install-repotools: go install ${REPOTOOLS_MODULE}/cmd/changelog@${REPOTOOLS_VERSION} set-smithy-go-version: @if [[ -z "${SMITHY_GO_VERSION}" ]]; then \ echo "SMITHY_GO_VERSION is required to update SDK's smithy-go module dependency version" && false; \ fi go run ${REPOTOOLS_CMD_EDIT_MODULE_DEPENDENCY} -s "github.com/aws/smithy-go" -v "${SMITHY_GO_VERSION}" external-changelog: mkdir -p .changelog cp changelog-template.json .changelog/00000000-0000-0000-0000-000000000000.json @echo "Generate a new UUID and update the file at .changelog/00000000-0000-0000-0000-000000000000.json" @echo "Make sure to rename the file with your new id, like .changelog/12345678-1234-1234-1234-123456789012.json" @echo "See CONTRIBUTING.md 'Changelog Documents' and an example at https://github.com/aws/aws-sdk-go-v2/pull/2934/files" ################## # Linting/Verify # ################## .PHONY: verify lint vet vet-modules-% sdkv1check verify: lint vet sdkv1check lint: @echo "go lint SDK and vendor packages" @lint=`golint ./...`; \ dolint=`echo "$$lint" | grep -E -v \ -e ${LINT_IGNORE_S3MANAGER_INPUT} \ -e ${LINTIGNORESINGLEFIGHT} \ -e ${LINT_IGNORE_AWSRULESFN_ARN} \ -e ${LINT_IGNORE_AWSRULESFN_PARTITION} \ -e ${LINT_IGNORE_PRIVATE_METRICS}`; \ echo "$$dolint"; \ if [ "$$dolint" != "" ]; then exit 1; fi vet: vet-modules-. vet-modules-%: @# See suffix-to-path pattern. Runs `go vet` cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst vet-modules-,,$@)) ${EACHMODULE_FLAGS} \ "go vet ${BUILD_TAGS} --all ./..." sdkv1check: @echo "Checking for usage of AWS SDK for Go v1" @sdkv1usage=`go list -test -f '''{{ if not .Standard }}{{ range $$_, $$name := .Imports }} * {{ $$.ImportPath }} -> {{ $$name }}{{ print "\n" }}{{ end }}{{ range $$_, $$name := .TestImports }} *: {{ $$.ImportPath }} -> {{ $$name }}{{ print "\n" }}{{ end }}{{ end}}''' ./... | sort -u | grep '''/aws-sdk-go/'''`; \ echo "$$sdkv1usage"; \ if [ "$$sdkv1usage" != "" ]; then exit 1; fi list-deps: list-deps-. list-deps-%: @# See suffix-to-path pattern. Lists dependencies for Go modules @cd ./internal/repotools/cmd/eachmodule \ && go run . -p $(subst _,/,$(subst list-deps-,,$@)) ${EACHMODULE_FLAGS} \ "go list -m all | grep -v 'github.com/aws/aws-sdk-go-v2'" | sort -u