stut/s3dir repository overview

# Clone the repository
git clone https://github.com/s3dir/s3dir
cd s3dir

# Build the binary
go build -o s3dir ./cmd/s3dir

# Run the server
./s3dir

# Start the server
./s3dir

# In another terminal, use the AWS CLI or any S3-compatible client
# Configure AWS CLI with dummy credentials (if auth is disabled)
aws configure set aws_access_key_id dummy
aws configure set aws_secret_access_key dummy

# Create a bucket
aws --endpoint-url=http://localhost:8000 s3 mb s3://my-bucket

# Upload a file
aws --endpoint-url=http://localhost:8000 s3 cp myfile.txt s3://my-bucket/

# List files
aws --endpoint-url=http://localhost:8000 s3 ls s3://my-bucket/

# Download a file
aws --endpoint-url=http://localhost:8000 s3 cp s3://my-bucket/myfile.txt downloaded.txt

# Delete a file
aws --endpoint-url=http://localhost:8000 s3 rm s3://my-bucket/myfile.txt

S3DIR_PORT=9000 S3DIR_VERBOSE=true ./s3dir

S3DIR_ENABLE_AUTH=true \
S3DIR_ACCESS_KEY_ID=myaccesskey \
S3DIR_SECRET_ACCESS_KEY=mysecretkey \
./s3dir

aws configure set aws_access_key_id myaccesskey
aws configure set aws_secret_access_key mysecretkey

S3DIR_READ_ONLY=true ./s3dir

# Start S3Dir
S3DIR_PORT=9000 ./s3dir

# Point your application to localhost:9000 instead of s3.amazonaws.com

# Start S3Dir in background
S3DIR_PORT=9000 ./s3dir &
S3DIR_PID=$!

# Run your tests
go test ./...

# Cleanup
kill $S3DIR_PID

# Copy your files to the data directory
mkdir -p data/website
cp -r public/* data/website/

# Start in read-only mode
S3DIR_DATA_DIR=data S3DIR_READ_ONLY=true ./s3dir

# Start S3Dir
S3DIR_DATA_DIR=/mnt/backups ./s3dir

# Use any S3 backup tool
restic -r s3:http://localhost:8000/backups init
restic -r s3:http://localhost:8000/backups backup /home

# List buckets
aws --endpoint-url=http://localhost:8000 s3 ls

# Sync a directory
aws --endpoint-url=http://localhost:8000 s3 sync ./local-dir s3://my-bucket/remote-dir/

# Copy with metadata
aws --endpoint-url=http://localhost:8000 s3 cp file.txt s3://my-bucket/ --metadata key1=value1,key2=value2

import boto3

# Create S3 client
s3 = boto3.client(
    's3',
    endpoint_url='http://localhost:8000',
    aws_access_key_id='dummy',
    aws_secret_access_key='dummy',
)

# Upload file
s3.upload_file('local-file.txt', 'my-bucket', 'remote-file.txt')

# Download file
s3.download_file('my-bucket', 'remote-file.txt', 'downloaded.txt')

# List objects
response = s3.list_objects_v2(Bucket='my-bucket')
for obj in response.get('Contents', []):
    print(obj['Key'])

package main

import (
    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/credentials"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3"
)

func main() {
    sess := session.Must(session.NewSession(&aws.Config{
        Endpoint:         aws.String("http://localhost:8000"),
        Region:           aws.String("us-east-1"),
        Credentials:      credentials.NewStaticCredentials("dummy", "dummy", ""),
        S3ForcePathStyle: aws.Bool(true),
    }))

    svc := s3.New(sess)

    // List buckets
    result, err := svc.ListBuckets(nil)
    if err != nil {
        panic(err)
    }

    for _, b := range result.Buckets {
        println(*b.Name)
    }
}

const AWS = require('aws-sdk');

const s3 = new AWS.S3({
    endpoint: 'http://localhost:8000',
    accessKeyId: 'dummy',
    secretAccessKey: 'dummy',
    s3ForcePathStyle: true,
    signatureVersion: 'v4',
});

// Upload file
s3.putObject({
    Bucket: 'my-bucket',
    Key: 'file.txt',
    Body: 'Hello, World!',
}, (err, data) => {
    if (err) console.error(err);
    else console.log('Upload successful:', data);
});

// List objects
s3.listObjectsV2({
    Bucket: 'my-bucket',
}, (err, data) => {
    if (err) console.error(err);
    else console.log('Objects:', data.Contents);
});

# Upload a large file using multipart upload (automatic)
aws --endpoint-url=http://localhost:8000 s3 cp large-file.bin s3://my-bucket/

# The AWS CLI automatically uses multipart upload for files larger than 8MB

import boto3

s3 = boto3.client(
    's3',
    endpoint_url='http://localhost:8000',
    aws_access_key_id='dummy',
    aws_secret_access_key='dummy',
)

# Automatic multipart upload for large files
s3.upload_file('large-file.bin', 'my-bucket', 'large-file.bin')

# Manual multipart upload
response = s3.create_multipart_upload(Bucket='my-bucket', Key='manual-upload.bin')
upload_id = response['UploadId']

# Upload parts
parts = []
with open('large-file.bin', 'rb') as f:
    part_number = 1
    while True:
        data = f.read(5 * 1024 * 1024)  # 5MB chunks
        if not data:
            break
        
        part = s3.upload_part(
            Bucket='my-bucket',
            Key='manual-upload.bin',
            PartNumber=part_number,
            UploadId=upload_id,
            Body=data
        )
        
        parts.append({
            'PartNumber': part_number,
            'ETag': part['ETag']
        })
        part_number += 1

# Complete the upload
s3.complete_multipart_upload(
    Bucket='my-bucket',
    Key='manual-upload.bin',
    UploadId=upload_id,
    MultipartUpload={'Parts': parts}
)

# Abort a multipart upload if needed
# s3.abort_multipart_upload(Bucket='my-bucket', Key='manual-upload.bin', UploadId=upload_id)

package main

import (
    "os"
    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/credentials"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3/s3manager"
)

func main() {
    sess := session.Must(session.NewSession(&aws.Config{
        Endpoint:         aws.String("http://localhost:8000"),
        Region:           aws.String("us-east-1"),
        Credentials:      credentials.NewStaticCredentials("dummy", "dummy", ""),
        S3ForcePathStyle: aws.Bool(true),
    }))

    uploader := s3manager.NewUploader(sess)

    file, err := os.Open("large-file.bin")
    if err != nil {
        panic(err)
    }
    defer file.Close()

    // Automatic multipart upload
    result, err := uploader.Upload(&s3manager.UploadInput{
        Bucket: aws.String("my-bucket"),
        Key:    aws.String("large-file.bin"),
        Body:   file,
    })
    
    if err != nil {
        panic(err)
    }
    
    println("Upload successful:", *result.Location)
}

┌─────────────────────────────────────┐
│         HTTP Handler (S3 API)        │
│  - Request parsing                   │
│  - XML response formatting           │
│  - Error handling                    │
└──────────────┬──────────────────────┘
               │
┌──────────────▼──────────────────────┐
│      Storage Layer (Filesystem)      │
│  - Bucket management                 │
│  - Object CRUD operations            │
│  - Directory traversal               │
└──────────────┬──────────────────────┘
               │
┌──────────────▼──────────────────────┐
│         Local Filesystem             │
│  - Buckets as directories            │
│  - Objects as files                  │
└─────────────────────────────────────┘

# Solution: Check directory permissions
chmod 755 ./data

# Solution: Use a different port
S3DIR_PORT=9000 ./s3dir

# Solution: Disable auth for local testing
S3DIR_ENABLE_AUTH=false ./s3dir

# Solution: Check if read-only mode is enabled
# Ensure S3DIR_READ_ONLY is not set to true

# Solution: Use prefix filtering to narrow results
aws --endpoint-url=http://localhost:8000 s3 ls s3://bucket/prefix/

# Solution: Use multipart uploads instead of single PUT requests
# AWS CLI automatically uses multipart for files >8MB:
aws --endpoint-url=http://localhost:8000 s3 cp large-file.bin s3://bucket/

# For other clients, configure multipart threshold:
# boto3: Set TransferConfig(multipart_threshold=...)
# AWS SDK v2: Use transfer manager with appropriate part size

# Why: Single PUT requests may buffer data in memory due to HTTP/TCP overhead.
# Multipart uploads use streaming for each part, keeping memory usage constant.

# Error: "Read timeout on endpoint URL"
# This happens during the final assembly step for large multipart uploads

# Solution: Increase the client timeout
# AWS CLI v2:
aws configure set s3.multipart_threshold 8MB
aws configure set s3.max_concurrent_requests 10
# Or set in environment:
export AWS_CLI_READ_TIMEOUT=300  # 5 minutes

# Python boto3:
from botocore.config import Config
config = Config(read_timeout=300)
s3 = boto3.client('s3', config=config, ...)

# Note: S3Dir optimizes assembly using:
# - 1MB buffer size for fast I/O
# - MD5-of-MD5s calculation (not full-file hash)
# - Typical assembly speed: ~500MB/sec on modern hardware