psjenkins/s3mover
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
Paul Jenkins
2026-03-26 11:06:15 +00:00
commit 20c1298a7f
14 changed files with 2859 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
backend/backend Executable file
View File

Binary file not shown.

28
backend/go.mod Normal file
View File

@@ -0,0 +1,28 @@
module s3mover/backend
go 1.26
require (
github.com/aws/aws-sdk-go-v2 v1.39.3
github.com/aws/aws-sdk-go-v2/config v1.31.13
github.com/aws/aws-sdk-go-v2/credentials v1.18.17
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.13
github.com/aws/aws-sdk-go-v2/service/s3 v1.88.5
)
require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.10 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.29.7 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.38.7 // indirect
github.com/aws/smithy-go v1.23.1 // indirect
)

38
backend/go.sum Normal file
View File

@@ -0,0 +1,38 @@
github.com/aws/aws-sdk-go-v2 v1.39.3 h1:h7xSsanJ4EQJXG5iuW4UqgP7qBopLpj84mpkNx3wPjM=
github.com/aws/aws-sdk-go-v2 v1.39.3/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko=
github.com/aws/aws-sdk-go-v2/config v1.31.13 h1:wcqQB3B0PgRPUF5ZE/QL1JVOyB0mbPevHFoAMpemR9k=
github.com/aws/aws-sdk-go-v2/config v1.31.13/go.mod h1:ySB5D5ybwqGbT6c3GszZ+u+3KvrlYCUQNo62+hkKOFk=
github.com/aws/aws-sdk-go-v2/credentials v1.18.17 h1:skpEwzN/+H8cdrrtT8y+rvWJGiWWv0DeNAe+4VTf+Vs=
github.com/aws/aws-sdk-go-v2/credentials v1.18.17/go.mod h1:Ed+nXsaYa5uBINovJhcAWkALvXw2ZLk36opcuiSZfJM=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10 h1:UuGVOX48oP4vgQ36oiKmW9RuSeT8jlgQgBFQD+HUiHY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10/go.mod h1:vM/Ini41PzvudT4YkQyE/+WiQJiQ6jzeDyU8pQKwCac=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.13 h1:9XV2TkOvCs6Fis10b4scQbv/eDPhklhU/65GikPxXAA=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.13/go.mod h1:X5gq64GsjuOIJRIUzR3x3Du96zUF+U1if3Qw/qNx1k8=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10 h1:mj/bdWleWEh81DtpdHKkw41IrS+r3uw1J/VQtbwYYp8=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10/go.mod h1:7+oEMxAZWP8gZCyjcm9VicI0M61Sx4DJtcGfKYv2yKQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10 h1:wh+/mn57yhUrFtLIxyFPh2RgxgQz/u+Yrf7hiHGHqKY=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10/go.mod h1:7zirD+ryp5gitJJ2m1BBux56ai8RIRDykXZrJSp540w=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.10 h1:FHw90xCTsofzk6vjU808TSuDtDfOOKPNdz5Weyc3tUI=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.10/go.mod h1:n8jdIE/8F3UYkg8O4IGkQpn2qUmapg/1K1yl29/uf/c=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.1 h1:ne+eepnDB2Wh5lHKzELgEncIqeVlQ1rSF9fEa4r5I+A=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.1/go.mod h1:u0Jkg0L+dcG1ozUq21uFElmpbmjBnhHR5DELHIme4wg=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10 h1:DRND0dkCKtJzCj4Xl4OpVbXZgfttY5q712H9Zj7qc/0=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10/go.mod h1:tGGNmJKOTernmR2+VJ0fCzQRurcPZj9ut60Zu5Fi6us=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.10 h1:DA+Hl5adieRyFvE7pCvBWm3VOZTRexGVkXw33SUqNoY=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.10/go.mod h1:L+A89dH3/gr8L4ecrdzuXUYd1znoko6myzndVGZx/DA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.88.5 h1:FlGScxzCGNzT+2AvHT1ZGMvxTwAMa6gsooFb1pO/AiM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.88.5/go.mod h1:N/iojY+8bW3MYol9NUMuKimpSbPEur75cuI1SmtonFM=
github.com/aws/aws-sdk-go-v2/service/sso v1.29.7 h1:fspVFg6qMx0svs40YgRmE7LZXh9VRZvTT35PfdQR6FM=
github.com/aws/aws-sdk-go-v2/service/sso v1.29.7/go.mod h1:BQTKL3uMECaLaUV3Zc2L4Qybv8C6BIXjuu1dOPyxTQs=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2 h1:scVnW+NLXasGOhy7HhkdT9AGb6kjgW7fJ5xYkUaqHs0=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2/go.mod h1:FRNCY3zTEWZXBKm2h5UBUPvCVDOecTad9KhynDyGBc0=
github.com/aws/aws-sdk-go-v2/service/sts v1.38.7 h1:VEO5dqFkMsl8QZ2yHsFDJAIZLAkEbaYDB+xdKi0Feic=
github.com/aws/aws-sdk-go-v2/service/sts v1.38.7/go.mod h1:L1xxV3zAdB+qVrVW/pBIrIAnHFWHo6FBbFe4xOGsG/o=
github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M=
github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=

388
backend/main.go Normal file
View File

@@ -0,0 +1,388 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"mime"
"net/http"
"net/url"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
type appConfig struct {
Addr string
Region string
Bucket string
Prefix string
FrontendOrigin string
EndpointURL string
UsePathStyle bool
}
type fileEntry struct {
Key string `json:"key"`
Size int64 `json:"size"`
LastModified time.Time `json:"lastModified"`
}
type s3Store struct {
client *s3.Client
uploader *manager.Uploader
bucket string
prefix string
}
func main() {
cfg, err := loadConfig()
if err != nil {
log.Fatal(err)
}
store, err := newS3Store(cfg)
if err != nil {
log.Fatal(err)
}
server := newServer(cfg, store)
log.Printf(
"storage config bucket=%q region=%q endpoint=%q path_style=%t prefix=%q",
cfg.Bucket,
cfg.Region,
cfg.EndpointURL,
cfg.UsePathStyle,
cfg.Prefix,
)
log.Printf("listening on %s", cfg.Addr)
if err := http.ListenAndServe(cfg.Addr, server); err != nil {
log.Fatal(err)
}
}
func loadConfig() (appConfig, error) {
cfg := appConfig{
Addr: envOrDefault("SERVER_ADDR", ":8080"),
Region: strings.TrimSpace(os.Getenv("AWS_REGION")),
Bucket: strings.TrimSpace(os.Getenv("S3_BUCKET")),
Prefix: strings.Trim(strings.TrimSpace(os.Getenv("S3_PREFIX")), "/"),
FrontendOrigin: strings.TrimSpace(os.Getenv("FRONTEND_ORIGIN")),
EndpointURL: strings.TrimSpace(os.Getenv("AWS_ENDPOINT_URL")),
UsePathStyle: strings.EqualFold(strings.TrimSpace(os.Getenv("AWS_USE_PATH_STYLE")), "true"),
}
if cfg.Region == "" {
return cfg, errors.New("AWS_REGION is required")
}
if cfg.Bucket == "" {
return cfg, errors.New("S3_BUCKET is required")
}
return cfg, nil
}
func newS3Store(cfg appConfig) (*s3Store, error) {
ctx := context.Background()
loadOptions := []func(*awsconfig.LoadOptions) error{
awsconfig.WithRegion(cfg.Region),
}
accessKey := strings.TrimSpace(os.Getenv("AWS_ACCESS_KEY_ID"))
secretKey := strings.TrimSpace(os.Getenv("AWS_SECRET_ACCESS_KEY"))
sessionToken := strings.TrimSpace(os.Getenv("AWS_SESSION_TOKEN"))
if accessKey != "" && secretKey != "" {
loadOptions = append(loadOptions, awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, sessionToken)))
}
awsCfg, err := awsconfig.LoadDefaultConfig(ctx, loadOptions...)
if err != nil {
return nil, fmt.Errorf("load aws config: %w", err)
}
clientOptions := func(o *s3.Options) {
o.UsePathStyle = cfg.UsePathStyle
if cfg.EndpointURL != "" {
o.BaseEndpoint = aws.String(cfg.EndpointURL)
}
}
client := s3.NewFromConfig(awsCfg, clientOptions)
return &s3Store{
client: client,
uploader: manager.NewUploader(client),
bucket: cfg.Bucket,
prefix: cfg.Prefix,
}, nil
}
func newServer(cfg appConfig, store *s3Store) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/api/health", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
})
mux.HandleFunc("/api/files", func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
handleListFiles(store, w, r)
case http.MethodPost:
handleUploadFile(store, w, r)
default:
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
})
mux.HandleFunc("/api/files/", func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
handleDownloadFile(store, w, r)
case http.MethodDelete:
handleDeleteFile(store, w, r)
default:
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
})
return withCORS(withLogging(mux), cfg.FrontendOrigin)
}
func handleListFiles(store *s3Store, w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
input := &s3.ListObjectsV2Input{
Bucket: aws.String(store.bucket),
Prefix: prefixValue(store.prefix),
}
var files []fileEntry
paginator := s3.NewListObjectsV2Paginator(store.client, input)
for paginator.HasMorePages() {
page, err := paginator.NextPage(ctx)
if err != nil {
writeAPIError(w, http.StatusBadGateway, "list files", err)
return
}
for _, item := range page.Contents {
if item.Key == nil {
continue
}
key := strings.TrimPrefix(aws.ToString(item.Key), store.prefix)
key = strings.TrimPrefix(key, "/")
if key == "" || strings.HasSuffix(key, "/") {
continue
}
files = append(files, fileEntry{
Key: key,
Size: aws.ToInt64(item.Size),
LastModified: aws.ToTime(item.LastModified),
})
}
}
sort.Slice(files, func(i, j int) bool {
return files[i].Key < files[j].Key
})
writeJSON(w, http.StatusOK, map[string]any{"files": files})
}
func handleUploadFile(store *s3Store, w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(64 << 20); err != nil {
writeAPIError(w, http.StatusBadRequest, "parse upload payload", err)
return
}
file, header, err := r.FormFile("file")
if err != nil {
writeAPIError(w, http.StatusBadRequest, "read upload file", errors.New("missing form field 'file'"))
return
}
defer file.Close()
key := sanitizeKey(r.FormValue("key"))
if key == "" {
key = sanitizeKey(header.Filename)
}
if key == "" {
writeAPIError(w, http.StatusBadRequest, "validate upload key", errors.New("file key is required"))
return
}
contentType := header.Header.Get("Content-Type")
if contentType == "" {
contentType = mime.TypeByExtension(filepath.Ext(key))
}
if contentType == "" {
contentType = "application/octet-stream"
}
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Minute)
defer cancel()
_, err = store.uploader.Upload(ctx, &s3.PutObjectInput{
Bucket: aws.String(store.bucket),
Key: aws.String(store.objectKey(key)),
Body: file,
ContentType: aws.String(contentType),
})
if err != nil {
writeAPIError(w, http.StatusBadGateway, "upload file", err)
return
}
writeJSON(w, http.StatusCreated, map[string]string{"key": key})
}
func handleDownloadFile(store *s3Store, w http.ResponseWriter, r *http.Request) {
key := objectKeyFromRequest(r)
if key == "" {
writeAPIError(w, http.StatusBadRequest, "read download key", errors.New("file key is required"))
return
}
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Minute)
defer cancel()
output, err := store.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(store.bucket),
Key: aws.String(store.objectKey(key)),
})
if err != nil {
var noSuchKey *types.NoSuchKey
if errors.As(err, &noSuchKey) {
writeAPIError(w, http.StatusNotFound, "download file", errors.New("file not found"))
return
}
writeAPIError(w, http.StatusBadGateway, "download file", err)
return
}
defer output.Body.Close()
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filepath.Base(key)))
if output.ContentType != nil {
w.Header().Set("Content-Type", aws.ToString(output.ContentType))
} else {
w.Header().Set("Content-Type", "application/octet-stream")
}
w.Header().Set("Content-Length", strconv.FormatInt(aws.ToInt64(output.ContentLength), 10))
if _, err := io.Copy(w, output.Body); err != nil {
log.Printf("stream download %q: %v", key, err)
}
}
func handleDeleteFile(store *s3Store, w http.ResponseWriter, r *http.Request) {
key := objectKeyFromRequest(r)
if key == "" {
writeAPIError(w, http.StatusBadRequest, "read delete key", errors.New("file key is required"))
return
}
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
_, err := store.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(store.bucket),
Key: aws.String(store.objectKey(key)),
})
if err != nil {
writeAPIError(w, http.StatusBadGateway, "delete file", err)
return
}
writeJSON(w, http.StatusOK, map[string]string{"deleted": key})
}
func (s *s3Store) objectKey(key string) string {
cleanKey := sanitizeKey(key)
if s.prefix == "" {
return cleanKey
}
return s.prefix + "/" + cleanKey
}
func objectKeyFromRequest(r *http.Request) string {
encodedKey := strings.TrimPrefix(r.URL.Path, "/api/files/")
decoded, err := url.PathUnescape(encodedKey)
if err != nil {
return ""
}
return sanitizeKey(decoded)
}
func sanitizeKey(value string) string {
parts := strings.Split(strings.TrimSpace(value), "/")
cleaned := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part == "" || part == "." || part == ".." {
continue
}
cleaned = append(cleaned, part)
}
return strings.Join(cleaned, "/")
}
func prefixValue(prefix string) *string {
if prefix == "" {
return nil
}
value := prefix + "/"
return &value
}
func envOrDefault(name, fallback string) string {
if value := strings.TrimSpace(os.Getenv(name)); value != "" {
return value
}
return fallback
}
func writeJSON(w http.ResponseWriter, status int, payload any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
if err := json.NewEncoder(w).Encode(payload); err != nil {
log.Printf("write json: %v", err)
}
}
func writeAPIError(w http.ResponseWriter, status int, operation string, err error) {
message := fmt.Sprintf("%s: %v", operation, err)
log.Printf("error %s", message)
writeJSON(w, status, map[string]string{"error": message})
}
func withLogging(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
next.ServeHTTP(w, r)
log.Printf("%s %s %s", r.Method, r.URL.Path, time.Since(start).Round(time.Millisecond))
})
}
func withCORS(next http.Handler, allowedOrigin string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if allowedOrigin != "" && r.Header.Get("Origin") == allowedOrigin {
w.Header().Set("Access-Control-Allow-Origin", allowedOrigin)
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
w.Header().Set("Access-Control-Allow-Methods", "GET,POST,DELETE,OPTIONS")
}
if r.Method == http.MethodOptions {
w.WriteHeader(http.StatusNoContent)
return
}
next.ServeHTTP(w, r)
})
}