Last active
December 21, 2021 14:04
-
-
Save niku/93addb6c2ca8aacde7b083c291cfdceb to your computer and use it in GitHub Desktop.
Make buckets has no object even if the bucket is versioning-enabled
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| module niku/cleanup-version-enabled-bucket | |
| go 1.17 | |
| require ( | |
| github.com/aws/aws-sdk-go-v2 v1.11.2 | |
| github.com/aws/aws-sdk-go-v2/config v1.11.0 | |
| github.com/aws/aws-sdk-go-v2/service/s3 v1.21.0 | |
| ) | |
| require ( | |
| github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.0.0 // indirect | |
| github.com/aws/aws-sdk-go-v2/credentials v1.6.4 // indirect | |
| github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 // indirect | |
| github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 // indirect | |
| github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 // indirect | |
| github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 // indirect | |
| github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.5.0 // indirect | |
| github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 // indirect | |
| github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.9.2 // indirect | |
| github.com/aws/aws-sdk-go-v2/service/sso v1.6.2 // indirect | |
| github.com/aws/aws-sdk-go-v2/service/sts v1.11.1 // indirect | |
| github.com/aws/smithy-go v1.9.0 // indirect | |
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package main | |
| import ( | |
| "context" | |
| "flag" | |
| "fmt" | |
| "strings" | |
| "github.com/aws/aws-sdk-go-v2/aws" | |
| "github.com/aws/aws-sdk-go-v2/config" | |
| "github.com/aws/aws-sdk-go-v2/service/s3" | |
| "github.com/aws/aws-sdk-go-v2/service/s3/types" | |
| ) | |
| func isTargetBucket(bucket types.Bucket) bool { | |
| // You can define target bucket | |
| return true | |
| } | |
| func collectTargetBuckets(client *s3.Client) ([]types.Bucket, error) { | |
| var result []types.Bucket | |
| listBucketsInput := s3.ListBucketsInput{} | |
| listBucketsOutput, err := client.ListBuckets(context.TODO(), &listBucketsInput) | |
| if err != nil { | |
| return result, err | |
| } | |
| for _, bucket := range listBucketsOutput.Buckets { | |
| if isTargetBucket(bucket) { | |
| result = append(result, bucket) | |
| } | |
| } | |
| return result, nil | |
| } | |
| func collectTargetObjectVersionsByEachBucket(client *s3.Client, input []types.Bucket) (map[string][]types.ObjectIdentifier, error) { | |
| result := make(map[string][]types.ObjectIdentifier) | |
| for _, bucket := range input { | |
| bucketName := *bucket.Name | |
| var keyMarker *string | |
| var versionIdMarker *string | |
| for { | |
| // It looks Maxkeys effects length of (Versions + DeleteMarkers). | |
| // i.e., If object versions in a bucket are over 1000, ListObjectVersions returns N Versions and 1000 - N DeleteMakers. | |
| listObjectVersionsInput := s3.ListObjectVersionsInput{ | |
| Bucket: aws.String(bucketName), | |
| KeyMarker: keyMarker, | |
| VersionIdMarker: versionIdMarker, | |
| } | |
| listObjectVersionsOutput, err := client.ListObjectVersions(context.TODO(), &listObjectVersionsInput) | |
| if err != nil { | |
| return result, err | |
| } | |
| // To make a versioning bucket has no object, it needs deleting all Versions and DeleteMarkers. | |
| for _, version := range listObjectVersionsOutput.Versions { | |
| objectIdentifier := types.ObjectIdentifier{ | |
| Key: version.Key, | |
| VersionId: version.VersionId, | |
| } | |
| result[bucketName] = append(result[bucketName], objectIdentifier) | |
| } | |
| for _, deleteMarker := range listObjectVersionsOutput.DeleteMarkers { | |
| objectIdentifier := types.ObjectIdentifier{ | |
| Key: deleteMarker.Key, | |
| VersionId: deleteMarker.VersionId, | |
| } | |
| result[bucketName] = append(result[bucketName], objectIdentifier) | |
| } | |
| keyMarker = listObjectVersionsOutput.NextKeyMarker | |
| versionIdMarker = listObjectVersionsOutput.NextVersionIdMarker | |
| count := len(result[bucketName]) | |
| fmt.Printf("%s:%d\n", bucketName, count) | |
| if keyMarker == nil && versionIdMarker == nil { | |
| break | |
| } | |
| } | |
| } | |
| return result, nil | |
| } | |
| func makeDeleteObjectsInputIncludingVersion(client *s3.Client, input map[string][]types.ObjectIdentifier) []s3.DeleteObjectsInput { | |
| var result []s3.DeleteObjectsInput | |
| for bucketName, objectVersions := range input { | |
| // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html | |
| // The request contains a list of up to 1000 keys that you want to delete. | |
| batchSize := 1000 | |
| for i := 0; i < len(objectVersions); i += batchSize { | |
| j := i + batchSize | |
| if len(objectVersions) < j { | |
| j = len(objectVersions) | |
| } | |
| var objectIdentifiers []types.ObjectIdentifier | |
| for _, objectVersion := range objectVersions[i:j] { | |
| // To delete versioned object, it needs both value Key and VersionId. | |
| objectIdentifier := types.ObjectIdentifier{ | |
| Key: aws.String(*objectVersion.Key), | |
| VersionId: objectVersion.VersionId, | |
| } | |
| objectIdentifiers = append(objectIdentifiers, objectIdentifier) | |
| } | |
| delete := types.Delete{ | |
| Objects: objectIdentifiers, | |
| } | |
| deleteObjectsInput := s3.DeleteObjectsInput{ | |
| Bucket: aws.String(bucketName), | |
| Delete: &delete, | |
| } | |
| result = append(result, deleteObjectsInput) | |
| } | |
| } | |
| return result | |
| } | |
| func deleteObjects(client *s3.Client, deleteObjectsInput s3.DeleteObjectsInput) error { | |
| _, err := client.DeleteObjects(context.TODO(), &deleteObjectsInput) | |
| if err != nil { | |
| return err | |
| } | |
| return nil | |
| } | |
| func main() { | |
| var deleteConfirmed bool | |
| flag.BoolVar(&deleteConfirmed, "delete-confirmed", false, "if you confirm deleting objects including versions, make this flug true. otherwise it runs dry-run.") | |
| flag.Parse() | |
| cfg, err := config.LoadDefaultConfig(context.TODO()) | |
| if err != nil { | |
| panic("configuration error, " + err.Error()) | |
| } | |
| client := s3.NewFromConfig(cfg) | |
| targetBuckets, err := collectTargetBuckets(client) | |
| if err != nil { | |
| fmt.Println("collectTargetBuckets err: " + err.Error()) | |
| panic(err) | |
| } | |
| targetObjectVersionsByEachBucket, err := collectTargetObjectVersionsByEachBucket(client, targetBuckets) | |
| if err != nil { | |
| fmt.Println("collectTargetObjectVersionsByEachBucket err: " + err.Error()) | |
| panic(err) | |
| } | |
| for k, v := range targetObjectVersionsByEachBucket { | |
| fmt.Printf("%s:%d\n", k, len(v)) | |
| } | |
| deleteObjectsInputs := makeDeleteObjectsInputIncludingVersion(client, targetObjectVersionsByEachBucket) | |
| for times, deleteObjectsInput := range deleteObjectsInputs { | |
| fmt.Printf("%d, %s, %d\n", times, *deleteObjectsInput.Bucket, len(deleteObjectsInput.Delete.Objects)) | |
| if deleteConfirmed { | |
| if err := deleteObjects(client, deleteObjectsInput); err != nil { | |
| fmt.Println("Got an error deleteing objects:" + err.Error()) | |
| } | |
| } | |
| } | |
| fmt.Println("Finished") | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment