fix(multiple): december 2024 wave (#589)

* fix(multiple): december 2024 wave #584

* fix(now context): caddyserver/cache-handler#116

* fix(surrogate-key): unescape if the key contains % #583

* fix(rfc): Last-Modified handling #588

* feat(caddy): handle API directly using the caddy admin api, solves #585

* fix(ci): remove useless ttl check

* fix(middleware): registering mapping eviction and run every minutes to purge IDX_ keys

Author: darkweak

context/now.go

@@ -19,8 +19,16 @@ func (*nowContext) SetContextWithBaseRequest(req *http.Request, _ *http.Request)
 func (cc *nowContext) SetupContext(_ configurationtypes.AbstractConfigurationInterface) {}
 
 func (cc *nowContext) SetContext(req *http.Request) *http.Request {
-	now := time.Now().UTC()
-	req.Header.Set("Date", now.Format(time.RFC1123))
+	var now time.Time
+	var e error
+
+	now, e = time.Parse(time.RFC1123, req.Header.Get("Date"))
+
+	if e != nil {
+		now = time.Now().UTC()
+		req.Header.Set("Date", now.Format(time.RFC1123))
+	}
+
 	return req.WithContext(context.WithValue(req.Context(), Now, now))
 }
 

pkg/middleware/middleware.go

@@ -50,6 +50,19 @@ func reorderStorers(storers []types.Storer, expectedStorers []string) []types.St
 	return newStorers
 }
 
+func registerMappingKeysEviction(logger core.Logger, storers []types.Storer) {
+	for _, storer := range storers {
+		logger.Debugf("registering mapping eviction for storer %s", storer.Name())
+		go func(current types.Storer) {
+			for {
+				logger.Debugf("run mapping eviction for storer %s", current.Name())
+				current.MapKeys(core.MappingKeyPrefix)
+				time.Sleep(time.Minute)
+			}
+		}(storer)
+	}
+}
+
 func NewHTTPCacheHandler(c configurationtypes.AbstractConfigurationInterface) *SouinBaseHandler {
 	if c.GetLogger() == nil {
 		var logLevel zapcore.Level
@@ -138,6 +151,8 @@ func NewHTTPCacheHandler(c configurationtypes.AbstractConfigurationInterface) *S
 	}
 	c.GetLogger().Info("Souin configuration is now loaded.")
 
+	registerMappingKeysEviction(c.GetLogger(), storers)
+
 	return &SouinBaseHandler{
 		Configuration:            c,
 		Storers:                  storers,
@@ -556,6 +571,17 @@ func (s *SouinBaseHandler) Revalidate(validator *core.Revalidator, next handlerF
 				return nil, errors.New("")
 			}
 
+			if validator.IfModifiedSincePresent {
+				if lastModified, err := time.Parse(time.RFC1123, customWriter.Header().Get("Last-Modified")); err == nil && validator.IfModifiedSince.Sub(lastModified) > 0 {
+					customWriter.handleBuffer(func(b *bytes.Buffer) {
+						b.Reset()
+					})
+					customWriter.Rw.WriteHeader(http.StatusNotModified)
+
+					return nil, errors.New("")
+				}
+			}
+
 			if statusCode != http.StatusNotModified {
 				err = s.Store(customWriter, rq, requestCc, cachedKey, uri)
 			}
@@ -922,10 +948,10 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n
 	case <-req.Context().Done():
 		switch req.Context().Err() {
 		case baseCtx.DeadlineExceeded:
-			customWriter.WriteHeader(http.StatusGatewayTimeout)
-			s.Configuration.GetLogger().Infof("Internal server error on endpoint %s: %v", req.URL, s.Storers)
 			rw.Header().Set("Cache-Status", cacheName+"; fwd=bypass; detail=DEADLINE-EXCEEDED")
+			customWriter.Rw.WriteHeader(http.StatusGatewayTimeout)
 			_, _ = customWriter.Rw.Write([]byte("Internal server error"))
+			s.Configuration.GetLogger().Infof("Internal server error on endpoint %s: %v", req.URL, s.Storers)
 			return baseCtx.DeadlineExceeded
 		case baseCtx.Canceled:
 			return baseCtx.Canceled

pkg/surrogate/providers/common.go

@@ -81,7 +81,7 @@ func uniqueTag(values []string) []string {
 		if _, found := tmp[item]; !found {
 			tmp[item] = true
 
-			if strings.Contains(item, "%3B") || strings.Contains(item, "%3A") {
+			if strings.Contains(item, "%") {
 				item, _ = url.QueryUnescape(item)
 			}
 			list = append(list, item)
@@ -178,6 +178,7 @@ func (*baseStorage) candidateStore(tag string) bool {
 
 func (*baseStorage) getOrderedSurrogateKeyHeadersCandidate() []string {
 	return []string{
+		cacheGroupKey,
 		surrogateKey,
 		edgeCacheTag,
 		cacheTags,

plugins/caddy/admin.go

@@ -0,0 +1,89 @@
+package httpcache
+
+import (
+	"fmt"
+	"github.com/caddyserver/caddy/v2"
+	"github.com/darkweak/souin/configurationtypes"
+	"github.com/darkweak/souin/pkg/api"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/darkweak/storages/core"
+)
+
+func init() {
+	caddy.RegisterModule(new(adminAPI))
+}
+
+// adminAPI is a module that serves PKI endpoints to retrieve
+// information about the CAs being managed by Caddy.
+type adminAPI struct {
+	ctx                      caddy.Context
+	logger                   core.Logger
+	app                      *SouinApp
+	InternalEndpointHandlers *api.MapHandler
+}
+
+// CaddyModule returns the Caddy module information.
+func (adminAPI) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "admin.api.souin",
+		New: func() caddy.Module { return new(adminAPI) },
+	}
+}
+
+func (a *adminAPI) handleAPIEndpoints(writer http.ResponseWriter, request *http.Request) error {
+	if a.InternalEndpointHandlers != nil {
+		for k, handler := range *a.InternalEndpointHandlers.Handlers {
+			if strings.Contains(request.RequestURI, k) {
+				handler(writer, request)
+				return nil
+			}
+		}
+	}
+
+	return caddy.APIError{
+		HTTPStatus: http.StatusNotFound,
+		Err:        fmt.Errorf("resource not found: %v", request.URL.Path),
+	}
+}
+
+// Provision sets up the adminAPI module.
+func (a *adminAPI) Provision(ctx caddy.Context) error {
+	a.ctx = ctx
+	a.logger = ctx.Logger(a).Sugar()
+
+	app, err := ctx.App(moduleName)
+	if err != nil {
+		return err
+	}
+
+	a.app = app.(*SouinApp)
+	config := Configuration{
+		API: a.app.API,
+		DefaultCache: DefaultCache{
+			TTL: configurationtypes.Duration{
+				Duration: 120 * time.Second,
+			},
+		},
+	}
+	a.InternalEndpointHandlers = api.GenerateHandlerMap(&config, a.app.Storers, a.app.SurrogateStorage)
+
+	return nil
+}
+
+// Routes returns the admin routes for the PKI app.
+func (a *adminAPI) Routes() []caddy.AdminRoute {
+	basepath := "/souin-api"
+	if a.app != nil && a.app.API.BasePath != "" {
+		basepath = a.app.API.BasePath
+	}
+
+	return []caddy.AdminRoute{
+		{
+			Pattern: basepath + "/{params...}",
+			Handler: caddy.AdminHandlerFunc(a.handleAPIEndpoints),
+		},
+	}
+}

plugins/caddy/app.go

@@ -1,8 +1,6 @@
 package httpcache
 
 import (
-	"errors"
-
 	"github.com/caddyserver/caddy/v2"
 	"github.com/darkweak/souin/configurationtypes"
 	"github.com/darkweak/souin/pkg/storage/types"
@@ -15,7 +13,7 @@ type SouinApp struct {
 	DefaultCache
 	// The provider to use.
 	Storers []types.Storer
-	// Surrogate storage to support th econfiguration reload without surrogate-key data loss.
+	// Surrogate storage to support the configuration reload without surrogate-key data loss.
 	SurrogateStorage providers.SurrogateInterface
 	// Cache-key tweaking.
 	CacheKeys configurationtypes.CacheKeys `json:"cache_keys,omitempty"`
@@ -39,9 +37,7 @@ func (s SouinApp) Start() error {
 	core.ResetRegisteredStorages()
 	_, _ = up.Delete(stored_providers_key)
 	_, _ = up.LoadOrStore(stored_providers_key, newStorageProvider())
-	if s.DefaultCache.GetTTL() == 0 {
-		return errors.New("Invalid/Incomplete default cache declaration")
-	}
+
 	return nil
 }
 

plugins/caddy/httpcache_test.go

@@ -1281,3 +1281,62 @@ func TestAllowedAdditionalStatusCode(t *testing.T) {
 		t.Error("Age header should be present")
 	}
 }
+
+type testTimeoutHandler struct {
+	iterator int
+}
+
+func (t *testTimeoutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	t.iterator++
+	if t.iterator%2 == 0 {
+		time.Sleep(5 * time.Second)
+
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+	_, _ = w.Write([]byte("Hello timeout!"))
+}
+
+func TestTimeout(t *testing.T) {
+	tester := caddytest.NewTester(t)
+	tester.InitServer(`
+	{
+		admin localhost:2999
+		http_port     9080
+		cache {
+			ttl 1ns
+			stale 1ns
+			timeout {
+				backend 1s
+			}
+		}
+	}
+	localhost:9080 {
+		route /cache-timeout {
+			cache
+			reverse_proxy localhost:9086
+		}
+	}`, "caddyfile")
+
+	go func() {
+		errorHandler := testTimeoutHandler{}
+		_ = http.ListenAndServe(":9086", &errorHandler)
+	}()
+	time.Sleep(time.Second)
+	resp1, _ := tester.AssertGetResponse(`http://localhost:9080/cache-timeout`, http.StatusOK, "Hello timeout!")
+	time.Sleep(time.Millisecond)
+	resp2, _ := tester.AssertGetResponse(`http://localhost:9080/cache-timeout`, http.StatusGatewayTimeout, "Internal server error")
+
+	if resp1.Header.Get("Cache-Status") != "Souin; fwd=uri-miss; stored; key=GET-http-localhost:9080-/cache-timeout" {
+		t.Errorf("unexpected resp1 Cache-Status header %v", resp1.Header.Get("Cache-Status"))
+	}
+
+	if resp1.Header.Get("Age") != "" {
+		t.Errorf("unexpected resp1 Age header %v", resp1.Header.Get("Age"))
+	}
+
+	if resp2.Header.Get("Cache-Status") != "Souin; fwd=bypass; detail=DEADLINE-EXCEEDED" {
+		t.Errorf("unexpected resp2 Cache-Status header %v", resp2.Header.Get("Cache-Status"))
+	}
+}