Как скопировать файлы размером более 5 ГБ между ведрами S3 с помощью PowerShell

Question

Как скопировать файлы размером более 5 ГБ между ведрами S3 с помощью PowerShell?

Существует несколько примеров использования PowerShell для копирования файлов между ведрами AWS S3, но они использовали команду Copy-S3Object, которая равна limited to files less than 5GB.

Answer 1

Вы должны сначала download и установить AWS Tools for Windows PowerShell.

Это основано на C# example на сайте AWS.

$AWSCredentials = Get-AWSCredentials -ProfileName YourProfileName # use Get-AWSCredentials -ListProfiles 

$sourceBucket = " " #source Bucket 
$targetBucket = " " #target Bucket 
$sourceObjectKey = " " #source filename 
$targetObjectKey = " " #target filename 
$region="USEast1" #region 
$copyResponses = @() 

$s3Config = New-Object Amazon.S3.AmazonS3Config 
$s3Config.RegionEndpoint = [Amazon.RegionEndpoint]::$region 


$s3client = New-Object Amazon.S3.AmazonS3Client($AWSCredentials,$s3Config) 

$initiateRequest = New-Object Amazon.S3.Model.InitiateMultipartUploadRequest 
$initiateRequest.BucketName = $targetBucket 
$initiateRequest.Key = $targetObjectKey 

$initResponse = $s3client.InitiateMultipartUpload($initiateRequest) 
$uploadId = $initResponse.UploadId 
$uploadId 

$metadataRequest = New-Object Amazon.S3.Model.GetObjectMetadataRequest 
$metadataRequest.BucketName = $sourceBucket 
$metadataRequest.Key = $sourceObjectKey 

$metadataResponse = New-Object Amazon.S3.Model.GetObjectMetadataResponse 
$metadataResponse = $s3client.GetObjectMetadata($metadataRequest) 
$objectSize = $metadataResponse.ContentLength; # in bytes 
$objectSize 

$partSize = 1GB 
$objectSize/$partSize 

$bytePosition = 0 
for ($i = 1; $bytePosition -lt $objectSize; $i++) 
{ 
    $copyRequest = New-Object Amazon.S3.Model.CopyPartRequest 
    $copyRequest.DestinationBucket = $targetBucket 
    $copyRequest.DestinationKey = $targetObjectKey 
    $copyRequest.SourceBucket = $sourceBucket 
    $copyRequest.SourceKey = $sourceObjectKey 
    $copyRequest.UploadId = $uploadId 
    $copyRequest.FirstByte = $bytePosition 
    $copyRequest.LastByte = if ($bytePosition + $partSize - 1 -ge $objectSize) { $objectSize - 1} else {$bytePosition + $partSize - 1} 
    $copyRequest.PartNumber = $i 

    $copyResponses += $s3Client.CopyPart($copyRequest) 
    $bytePosition += $partSize      
} 

$completeRequest = New-Object Amazon.S3.Model.CompleteMultipartUploadRequest 
$completeRequest.BucketName = $targetBucket 
$completeRequest.Key = $targetObjectKey 
$completeRequest.UploadId = $uploadId 
$completeRequest.AddPartETags($copyResponses) 

$completeUploadResponse =$s3Client.CompleteMultipartUpload($completeRequest)