Merge pull request #27 from ConstellationCrypto/inomurko/path-trim-fix

Add S3 path, replace deprecated docker compose, trim origin block

Author: danyalprout

Makefile

@@ -5,7 +5,7 @@ build:
 .PHONY: build
 
 build-docker:
-	docker-compose build
+	docker compose build
 .PHONY: build-docker
 
 clean:
@@ -21,8 +21,8 @@ test:
 .PHONY: test
 
 integration:
-	docker-compose down
-	docker-compose up -d minio create-buckets
+	docker compose down
+	docker compose up -d minio create-buckets
 	RUN_INTEGRATION_TESTS=true go test -v ./...
 .PHONY: integration
 

README.md

@@ -46,7 +46,7 @@ To run the project locally, you should first copy `.env.template` to `.env` and
 to your beacon client and storage backend of choice. Then you can run the project with:
 
 ```sh
-docker-compose up
+docker compose up
 ```
 
 You can see a full list of configuration options by running:

archiver/flags/config.go

@@ -2,6 +2,7 @@ package flags
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	common "github.com/base-org/blob-archiver/common/flags"
@@ -35,7 +36,7 @@ func (c ArchiverConfig) Check() error {
 	}
 
 	if c.OriginBlock == (geth.Hash{}) {
-		return fmt.Errorf("invalid origin block")
+		return fmt.Errorf("invalid origin block %s", c.OriginBlock)
 	}
 
 	if c.ListenAddr == "" {
@@ -53,7 +54,7 @@ func ReadConfig(cliCtx *cli.Context) ArchiverConfig {
 		BeaconConfig:  common.NewBeaconConfig(cliCtx),
 		StorageConfig: common.NewStorageConfig(cliCtx),
 		PollInterval:  pollInterval,
-		OriginBlock:   geth.HexToHash(cliCtx.String(ArchiverOriginBlock.Name)),
+		OriginBlock:   geth.HexToHash(strings.Trim(cliCtx.String(ArchiverOriginBlock.Name), "\"")),
 		ListenAddr:    cliCtx.String(ArchiverListenAddrFlag.Name),
 	}
 }

common/flags/config.go

@@ -24,6 +24,7 @@ type S3Config struct {
 	Endpoint string
 	UseHttps bool
 	Bucket   string
+	Path     string
 
 	S3CredentialType S3CredentialType
 	AccessKey        string
@@ -106,6 +107,7 @@ func readS3Config(ctx *cli.Context) S3Config {
 		SecretAccessKey:  ctx.String(S3SecretAccessKeyFlagName),
 		UseHttps:         ctx.Bool(S3EndpointHttpsFlagName),
 		Bucket:           ctx.String(S3BucketFlagName),
+		Path:             ctx.String(S3PathFlagName),
 		S3CredentialType: toS3CredentialType(ctx.String(S3CredentialTypeFlagName)),
 		Compress:         ctx.Bool(S3CompressFlagName),
 	}

common/flags/flags.go

@@ -17,6 +17,7 @@ const (
 	S3AccessKeyFlagName             = "s3-access-key"
 	S3SecretAccessKeyFlagName       = "s3-secret-access-key"
 	S3BucketFlagName                = "s3-bucket"
+	S3PathFlagName                  = "s3-path"
 	FileStorageDirectoryFlagName    = "file-directory"
 )
 
@@ -77,6 +78,13 @@ func CLIFlags(envPrefix string) []cli.Flag {
 			Hidden:  true,
 			EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_BUCKET"),
 		},
+		&cli.StringFlag{
+			Name:    S3PathFlagName,
+			Usage:   "The path to append to file",
+			Hidden:  true,
+			EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_PATH"),
+			Value:   "",
+		},
 		// File Data Store Flags
 		&cli.StringFlag{
 			Name:    FileStorageDirectoryFlagName,

common/storage/s3.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"encoding/json"
 	"io"
+	"path"
 
 	"github.com/base-org/blob-archiver/common/flags"
 	"github.com/ethereum/go-ethereum/common"
@@ -17,6 +18,7 @@ import (
 type S3Storage struct {
 	s3       *minio.Client
 	bucket   string
+	path     string
 	log      log.Logger
 	compress bool
 }
@@ -41,13 +43,14 @@ func NewS3Storage(cfg flags.S3Config, l log.Logger) (*S3Storage, error) {
 	return &S3Storage{
 		s3:       client,
 		bucket:   cfg.Bucket,
+		path:     cfg.Path,
 		log:      l,
 		compress: cfg.Compress,
 	}, nil
 }
 
 func (s *S3Storage) Exists(ctx context.Context, hash common.Hash) (bool, error) {
-	_, err := s.s3.StatObject(ctx, s.bucket, hash.String(), minio.StatObjectOptions{})
+	_, err := s.s3.StatObject(ctx, s.bucket, path.Join(s.path, hash.String()), minio.StatObjectOptions{})
 	if err != nil {
 		errResponse := minio.ToErrorResponse(err)
 		if errResponse.Code == "NoSuchKey" {
@@ -61,7 +64,7 @@ func (s *S3Storage) Exists(ctx context.Context, hash common.Hash) (bool, error)
 }
 
 func (s *S3Storage) Read(ctx context.Context, hash common.Hash) (BlobData, error) {
-	res, err := s.s3.GetObject(ctx, s.bucket, hash.String(), minio.GetObjectOptions{})
+	res, err := s.s3.GetObject(ctx, s.bucket, path.Join(s.path, hash.String()), minio.GetObjectOptions{})
 	if err != nil {
 		s.log.Info("unexpected error fetching blob", "hash", hash.String(), "err", err)
 		return BlobData{}, ErrStorage
@@ -122,7 +125,7 @@ func (s *S3Storage) Write(ctx context.Context, data BlobData) error {
 
 	reader := bytes.NewReader(b)
 
-	_, err = s.s3.PutObject(ctx, s.bucket, data.Header.BeaconBlockHash.String(), reader, int64(len(b)), options)
+	_, err = s.s3.PutObject(ctx, s.bucket, path.Join(s.path, data.Header.BeaconBlockHash.String()), reader, int64(len(b)), options)
 
 	if err != nil {
 		s.log.Warn("error writing blob", "err", err)

common/storage/s3_test.go

@@ -14,8 +14,8 @@ import (
 
 // Prior to running these tests, a local Minio server must be running.
 // You can accomplish this with:
-// docker-compose down # shut down any running services
-// docker-compose up minio create-buckets # start the minio service
+// docker compose down # shut down any running services
+// docker compose up minio create-buckets # start the minio service
 func setupS3(t *testing.T) *S3Storage {
 	if os.Getenv("RUN_INTEGRATION_TESTS") == "" {
 		t.Skip("skipping integration tests: set RUN_INTEGRATION_TESTS environment variable")