Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[R2] Add AWS SDK checksum examples #19280

Open
wants to merge 5 commits into
base: production
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 106 additions & 5 deletions src/content/docs/r2/examples/aws/aws-sdk-go.mdx
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
---
title: aws-sdk-go
pcx_content_type: example

---

import { Render } from "~/components"
import { Render } from "~/components";

<Render file="keys" /><br/>
<Render file="keys" />
<br />

This example uses version 2 of the [aws-sdk-go](https://github.com/aws/aws-sdk-go-v2) package. You must pass in the R2 configuration credentials when instantiating your `S3` service client:

Expand All @@ -16,10 +16,11 @@ Client version `1.73.0` introduced a modification to the default checksum behavi
To mitigate, users can use `1.72.3` or add the following to their config:

```go
config.WithRequestChecksumCalculation(0)
config.WithResponseChecksumValidation(0)
config.WithRequestChecksumCalculation(2)
config.WithResponseChecksumValidation(2)
```

If checksum is needed, R2 supports `SHA-1` and `SHA-256` algorithms.
:::

```go
Expand Down Expand Up @@ -93,6 +94,106 @@ func main() {
}
```

### With SHA-1/SHA-256 checksum algorithms
Oxyjun marked this conversation as resolved.
Show resolved Hide resolved

You can also use SHA-1 and SHA-256 algorithms for checksum.

```go
package main

import (
"bytes"
"context"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"fmt"
"log"
"os"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)

func calculateSHA256(data []byte) string {
hash := sha256.New()
hash.Write(data)
return base64.StdEncoding.EncodeToString(hash.Sum(nil))
}

func calculateSHA1(data []byte) string {
hash := sha1.New()
hash.Write(data)
return base64.StdEncoding.EncodeToString(hash.Sum(nil))
}

func main() {
var bucketName = "sdk-example"
var accountId = "<accountid>"
var accessKeyId = "<access_key_id>"
var accessKeySecret = "<access_key_secret>"

cfg, err := config.LoadDefaultConfig(context.TODO(),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyId, accessKeySecret, "")),
config.WithRegion("auto"),
)
if err != nil {
log.Fatal(err)
}

client := s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(fmt.Sprintf("https://%s.r2.cloudflarestorage.com", accountId))
})

// Read your object data (example with a file)
data, err := os.ReadFile("/path/to/file")
if err != nil {
log.Fatal(err)
}

// Calcualte SHA1
checksumSHA1 := calculateSHA1(data)

// Create the upload input
uploadInputForSHA1 := &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: bytes.NewReader(data),
ChecksumSHA1: aws.String(checksumSHA1),
}

// Upload the object with SHA1 checksum
resultSHA1, errSHA1 := client.PutObject(context.TODO(), uploadInputForSHA1)
if errSHA1 != nil {
log.Fatal(errSHA1)
}

fmt.Printf("Upload successful with SHA1 checksum: %+v\n", resultSHA1)

// Calculate SHA256
checksumSHA256 := calculateSHA256(data)

// Create the upload input
uploadInputForSHA256 := &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: bytes.NewReader(data),
ChecksumSHA256: aws.String(checksumSHA256),
}

// Upload the object with SHA256 checksum
resultSHA256, errSHA256 := client.PutObject(context.TODO(), uploadInputForSHA256)
if errSHA256 != nil {
log.Fatal(errSHA256)
}

fmt.Printf("Upload successful with SHA256 checksum: %+v\n", resultSHA256)

}

```

## Generate presigned URLs

You can also generate presigned links that can be used to temporarily share public write access to a bucket.
Expand Down
90 changes: 90 additions & 0 deletions src/content/docs/r2/examples/aws/aws-sdk-js-v3.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ requestChecksumCalculation: "WHEN_REQUIRED",
responseChecksumValidation: "WHEN_REQUIRED",
```

If checksum is needed, R2 supports `SHA-1` and `SHA-256` algorithms.
:::

```ts
Expand Down Expand Up @@ -106,6 +107,95 @@ console.log(
// }
```

### With SHA-1/SHA-256 checksum algorithms
Oxyjun marked this conversation as resolved.
Show resolved Hide resolved

You can also use SHA-1 and SHA-256 algorithms for checksum.

```ts
import {
S3Client,
ListBucketsCommand,
ListObjectsV2Command,
GetObjectCommand,
PutObjectCommand,
} from "@aws-sdk/client-s3";
import { createHash } from "node:crypto";

const S3 = new S3Client({
region: "auto",
endpoint: `https://${ACCOUNT_ID}.r2.cloudflarestorage.com`,
credentials: {
accessKeyId: ACCESS_KEY_ID,
secretAccessKey: SECRET_ACCESS_KEY,
},
});

// Create an array buffer
const arrayBuffer = await OBJECT.arrayBuffer();

// Create SHA-1 hash of the object to upload
const ChecksumSHA1 = createHash("sha256")
.update(Buffer.from(arrayBuffer))
.digest("base64");

// Upload the object with the checksums
console.log(
await S3.send(
new PutObjectCommand({
Bucket: BUCKET_NAME,
Key: OBJECT_KEY,
Body: OBJECT,
ChecksumSHA1,
}),
),
);

// {
// '$metadata': {
// httpStatusCode: 200,
// requestId: undefined,
// extendedRequestId: undefined,
// cfId: undefined,
// attempts: 1,
// totalRetryDelay: 0
// },
// ETag: '"355801ab7ccb9ffddcb4c47e8cd61584"',
// ChecksumSHA1: '6MMnUIGMVR/u6AO3uCoUcSRnmzQ=',
// VersionId: '7e6b8ae6e2198a8c1acf76598af339ef'
// }

// Create SHA-256 hash of the object to upload
const ChecksumSHA256 = createHash("sha256")
.update(Buffer.from(arrayBuffer))
.digest("base64");

// Upload the object with the checksums
console.log(
await S3.send(
new PutObjectCommand({
Bucket: BUCKET_NAME,
Key: OBJECT_KEY,
Body: OBJECT,
ChecksumSHA256,
}),
),
);

// {
// '$metadata': {
// httpStatusCode: 200,
// requestId: undefined,
// extendedRequestId: undefined,
// cfId: undefined,
// attempts: 1,
// totalRetryDelay: 0
// },
// ETag: '"f0d8680d5c596202dd81afa17428c65f"',
// ChecksumSHA256: 'jSIKqrDnDlJg3pSnXflg9QJyzGiexsvIa3tCCRfb3DA=',
// VersionId: '7e6b8ae42793fb4a693f020ff58ef8d0'
// }
```

## Generate presigned URLs

You can also generate presigned links that can be used to share public read or write access to a bucket temporarily.
Expand Down
53 changes: 51 additions & 2 deletions src/content/docs/r2/examples/aws/boto3.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,11 @@ Client version `1.36.0` introduced a modification to the default checksum behavi
To mitigate, users can use `1.35.99` or add the following to their s3 resource config:

```python
request_checksum_calculation = 'WHEN_REQUIRED',
response_checksum_validation = 'WHEN_REQUIRED'
request_checksum_calculation = 'when_required',
response_checksum_validation = 'when_required'
```

If checksum is needed, R2 supports `SHA-1` and `SHA-256` algorithms.
:::

```python
Expand Down Expand Up @@ -69,3 +70,51 @@ Objects:
- cat.png
- todos.txt
```

### With SHA-1/SHA-256 checksum algorithms
Oxyjun marked this conversation as resolved.
Show resolved Hide resolved

You can also use SHA-1 and SHA-256 algorithms for checksum.

```python
import boto3
import hashlib
import base64

def calculate_sha1_base64(data):
"""Calculate SHA-1 hash and return base64 encoded string."""
sha1_hash = hashlib.sha1(data).digest()
return base64.b64encode(sha1_hash).decode('utf-8')

def calculate_sha256_base64(data):
"""Calculate SHA-256 hash and return base64 encoded string."""
sha256_hash = hashlib.sha256(data).digest()
return base64.b64encode(sha256_hash).decode('utf-8')

s3 = boto3.client(
service_name ="s3",
endpoint_url = 'https://<accountid>.r2.cloudflarestorage.com',
aws_access_key_id = '<access_key_id>',
aws_secret_access_key = '<access_key_secret>',
region_name="<location>", # Must be one of: wnam, enam, weur, eeur, apac, auto
)

# Calculate SHA1
sha1 = calculate_sha1_base64(file_data)

# Uplodad file to R2 with SHA1 checksum
response = s3.put_object(
Body=file_data,
Bucket=bucket_name,
Key='sha1.txt',
ChecksumSHA1=sha1)

# Calculate SHA256
sha256 = calculate_sha256_base64(file_data)

# Uplodad file to R2 with SHA256 checksum
response = s3.put_object(
Body=file_data,
Bucket=bucket_name,
Key='sha256.txt',
ChecksumSHA256=sha256)
```
Loading