fix: raw response to response process with 'interrogation id' duplicates

.github/workflows/docker.yaml

@@ -3,7 +3,8 @@ name: Build snapshot docker image
 on:
   push:
     branches-ignore:
-      - main
+      - * # main
+      # turned off for now (needs secrets renewal)
 
 jobs:
   build-snapshot:

src/main/java/fr/insee/genesis/controller/exception/ExceptionController.java

@@ -0,0 +1,75 @@
+package fr.insee.genesis.controller.exception;
+
+import fr.insee.genesis.exceptions.*;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ProblemDetail;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+
+/**
+ * This controller uses Spring's ControllerAdvice annotation to intercept exceptions.
+ * It implements the <a href="https://www.rfc-editor.org/rfc/rfc9457.html">RFC 9457</a> by returning
+ * Spring's <code>ProblemDetail</code> object.
+ */
+@ControllerAdvice
+@Slf4j
+public class ExceptionController {
+
+    // Note: No handler for uncaught Exception.class for now since it breaks soms tests.
+
+    @ExceptionHandler
+    public ProblemDetail handleGenericGenesisException(GenesisException genesisException) {
+        log.error("GenesisException: {}", genesisException.getMessage(), genesisException);
+        return ProblemDetail.forStatusAndDetail(
+                resolveHttpCode(genesisException.getStatus()),
+                genesisException.getMessage());
+    }
+
+    /** Returns the corresponding http status, or 500 if the given code does not match a http status. */
+    private static HttpStatus resolveHttpCode(int statusCode) {
+        HttpStatus httpStatus = HttpStatus.resolve(statusCode);
+        return httpStatus != null ? httpStatus : HttpStatus.INTERNAL_SERVER_ERROR;
+    }
+
+    @ExceptionHandler(InvalidDateIntervalException.class)
+    public ProblemDetail handleInvalidDateIntervalException(InvalidDateIntervalException e) {
+        log.error("InvalidDateIntervalException: {}", e.getMessage());
+        return ProblemDetail.forStatusAndDetail(
+                HttpStatus.BAD_REQUEST,
+                e.getMessage());
+    }
+
+    @ExceptionHandler(ModesConflictException.class)
+    public ProblemDetail handleModesConflictException(ModesConflictException e) {
+        log.error("ModesConflictException: {}", e.getMessage());
+        return ProblemDetail.forStatusAndDetail(
+                HttpStatus.CONFLICT,
+                e.getMessage());
+    }
+
+    @ExceptionHandler(UndefinedModesException.class)
+    public ProblemDetail handleUndefinedModesException(UndefinedModesException e) {
+        log.error("UndefinedModesException: {}", e.getMessage());
+        return ProblemDetail.forStatusAndDetail(
+                HttpStatus.NOT_FOUND,
+                e.getMessage());
+    }
+
+    @ExceptionHandler(UndefinedMetadataException.class)
+    public ProblemDetail handleUndefinedMetadataException(UndefinedMetadataException e) {
+        log.error("UndefinedMetadataException: {}", e.getMessage());
+        return ProblemDetail.forStatusAndDetail(
+                HttpStatus.NOT_FOUND,
+                e.getMessage());
+    }
+
+    @ExceptionHandler(InvalidMetadataException.class)
+    public ProblemDetail handleInvalidMetadataException(InvalidMetadataException e) {
+        log.error("InvalidMetadataException: {}", e.getMessage());
+        return ProblemDetail.forStatusAndDetail(
+                HttpStatus.BAD_REQUEST,
+                e.getMessage());
+    }
+
+}

src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java

@@ -15,6 +15,7 @@
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.util.List;
 
@@ -55,17 +56,17 @@ public ResponseEntity<List<InterrogationId>> getAllInterrogationIdsByCollectionI
             @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
             @Parameter(
                     description = "sinceDate",
-                    schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00")
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00Z")
             )
-            LocalDateTime start,
+            Instant start,
 
             @RequestParam("end")
             @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
             @Parameter(
                     description = "untilDate",
-                    schema = @Schema(type = "string", format = "date-time", example = "2026-01-31T23:59:59")
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-01-31T23:59:59Z")
             )
-            LocalDateTime end) {
+            Instant end) {
         List<InterrogationId> responses = surveyUnitService.findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(collectionInstrumentId, start,end);
         return ResponseEntity.ok(responses);
     }

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java

@@ -15,6 +15,7 @@
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import jakarta.validation.Valid;
+import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
@@ -43,23 +44,16 @@
 
 @Slf4j
 @Controller
+@RequiredArgsConstructor
 public class RawResponseController {
 
     private static final String SUCCESS_MESSAGE = "Interrogation %s saved";
     private static final String INTERROGATION_ID = "interrogationId";
-    public static final String NB_DOCS_WITH_FORMATTED = "%d document(s) processed, including %d FORMATTED after data verification for collectionInstrumentId %s";
-    public static final String NB_DOCS = "%d document(s) processed for collectionInstrumentId %s";
+
     private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort;
     private final RawResponseApiPort rawResponseApiPort;
     private final RawResponseInputRepository rawRepository;
 
-
-    public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort, RawResponseApiPort rawResponseApiPort, RawResponseInputRepository rawRepository) {
-        this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
-        this.rawResponseApiPort = rawResponseApiPort;
-        this.rawRepository = rawRepository;
-    }
-
     @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database")
     @PutMapping(path = "/responses/raw/lunatic-json/save")
     @PreAuthorize("hasRole('COLLECT_PLATFORM')")
@@ -118,11 +112,8 @@ public ResponseEntity<String> processRawResponses(
         log.info("Try to process raw responses for collectionInstrumentId {} and {} interrogationIds", collectionInstrumentId, interrogationIdList.size());
         List<GenesisError> errors = new ArrayList<>();
         try {
-            DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId, interrogationIdList, errors);
-            return result.formattedDataCount() == 0 ?
-                    ResponseEntity.ok(NB_DOCS.formatted(result.dataCount(), collectionInstrumentId))
-                    : ResponseEntity.ok(NB_DOCS_WITH_FORMATTED
-                    .formatted(result.dataCount(), result.formattedDataCount(), collectionInstrumentId));
+            DataProcessResult result = rawResponseApiPort.processRawResponsesByInterrogationIds(collectionInstrumentId, interrogationIdList, errors);
+            return ResponseEntity.ok(result.message(collectionInstrumentId));
         } catch (GenesisException e) {
             return ResponseEntity.status(e.getStatus()).body(e.getMessage());
         }
@@ -140,11 +131,8 @@ public ResponseEntity<String> processRawResponsesByCollectionInstrumentId(
     ) {
         log.info("Try to process raw responses for collectionInstrumentId {}", collectionInstrumentId);
         try {
-            DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId);
-            return result.formattedDataCount() == 0 ?
-                    ResponseEntity.ok(NB_DOCS.formatted(result.dataCount(), collectionInstrumentId))
-                    : ResponseEntity.ok(NB_DOCS_WITH_FORMATTED
-                    .formatted(result.dataCount(), result.formattedDataCount(), collectionInstrumentId));
+            DataProcessResult result = rawResponseApiPort.processRawResponsesByInterrogationIds(collectionInstrumentId);
+            return ResponseEntity.ok(result.message(collectionInstrumentId));
         } catch (GenesisException e) {
             return ResponseEntity.status(e.getStatus()).body(e.getMessage());
         }
@@ -183,7 +171,7 @@ public ResponseEntity<LunaticJsonRawDataModel> getJsonRawData(
             @RequestParam("campaignName") String campaignName,
             @RequestParam(value = "mode") Mode modeSpecified
     ) {
-        List<LunaticJsonRawDataModel> data = lunaticJsonRawDataApiPort.getRawData(campaignName, modeSpecified, List.of(interrogationId));
+        List<LunaticJsonRawDataModel> data = lunaticJsonRawDataApiPort.getRawDataByQuestionnaireId(campaignName, modeSpecified, List.of(interrogationId));
         return ResponseEntity.ok(data.getFirst());
     }
 
@@ -201,7 +189,7 @@ public ResponseEntity<String> processJsonRawData(
         List<GenesisError> errors = new ArrayList<>();
 
         try {
-            DataProcessResult result = lunaticJsonRawDataApiPort.processRawData(campaignName, interrogationIdList, errors);
+            DataProcessResult result = lunaticJsonRawDataApiPort.processRawDataByInterrogationIds(campaignName, interrogationIdList, errors);
             return result.formattedDataCount() == 0 ?
                     ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount()))
                     : ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification"

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseReprocessController.java

@@ -0,0 +1,102 @@
+package fr.insee.genesis.controller.rest.responses;
+
+import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
+import fr.insee.genesis.domain.ports.api.ReprocessRawResponseApiPort;
+import fr.insee.genesis.exceptions.GenesisException;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.format.annotation.DateTimeFormat;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+
+import java.time.Instant;
+
+@Controller
+@RequiredArgsConstructor
+@Slf4j
+public class RawResponseReprocessController {
+
+    private final ReprocessRawResponseApiPort reprocessRawResponseApiPort;
+
+    @Operation(summary = "Reprocess raw response of a collection instrument.")
+    @PostMapping(path = "/raw-responses/{collectionInstrumentId}/reprocess")
+    @PreAuthorize("hasRole('ADMIN')")
+    public ResponseEntity<String> reProcessRawResponsesByCollectionInstrumentId(
+            @Parameter(
+                    description = "Id of the collection instrument (old questionnaireId)",
+                    example = "ENQTEST2025X00")
+            @PathVariable("collectionInstrumentId")
+            String collectionInstrumentId,
+
+            @Parameter(
+                    description = "Extract since",
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00Z")
+            )
+            @RequestParam(value = "sinceDate", required = false)
+            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+            Instant sinceDate,
+
+            @Parameter(
+                    description = "Extract until",
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-02-02T00:00:00Z")
+            )
+            @RequestParam(value = "endDate", required = false)
+            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+            Instant endDate
+    ) throws GenesisException {
+
+        DataProcessResult result = reprocessRawResponseApiPort.reprocessRawResponses(
+                RawDataModelType.FILIERE,
+                collectionInstrumentId,
+                sinceDate,
+                endDate);
+
+        return ResponseEntity.ok(result.message(collectionInstrumentId));
+    }
+
+    @Operation(summary = "Reprocess Lunatic raw data for a questionnaire model. " +
+            "**Note**: Lunatic raw data is the legacy format of raw responses.")
+    @PostMapping(path = "/responses/raw/lunatic-json/{questionnaireId}/reprocess")
+    @PreAuthorize("hasRole('ADMIN')")
+    public ResponseEntity<String> reProcessJsonRawDataByQuestionnaireId(
+            @Parameter(
+                    description = "Questionnaire model id (old name for collection instrument id).",
+                    example = "ENQTEST2025X00")
+            @PathVariable("questionnaireId")
+            String collectionInstrumentId, // 'questionnaireId' is the legacy name for 'collectionInstrumentId'
+
+            @Parameter(
+                    description = "Extract since",
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00Z")
+            )
+            @RequestParam(value = "sinceDate", required = false)
+            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+            Instant sinceDate,
+
+            @Parameter(
+                    description = "Extract until",
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-02-02T00:00:00Z")
+            )
+            @RequestParam(value = "endDate", required = false)
+            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+            Instant endDate
+    ) throws GenesisException {
+
+        DataProcessResult result = reprocessRawResponseApiPort.reprocessRawResponses(
+                RawDataModelType.LEGACY,
+                collectionInstrumentId,
+                sinceDate,
+                endDate);
+
+        return ResponseEntity.ok(result.message(collectionInstrumentId));
+    }
+
+}

src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java

@@ -1,16 +1,18 @@
 package fr.insee.genesis.controller.utils;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.exceptions.ModesConflictException;
+import fr.insee.genesis.exceptions.UndefinedModesException;
+import fr.insee.genesis.infrastructure.utils.FileUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import fr.insee.genesis.domain.model.surveyunit.Mode;
-import fr.insee.genesis.exceptions.GenesisException;
-import fr.insee.genesis.infrastructure.utils.FileUtils;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+// Note: this class should be moved in the domain service layer.
 
 @Component
 @Slf4j
@@ -23,25 +25,23 @@ public ControllerUtils(FileUtils fileUtils) {
 		this.fileUtils = fileUtils;
 	}
 
-
 	/**
 	 * If a mode is specified, we treat only this mode.
-	 * If no mode is specified, we treat all modes in the campaign.
+	 * If no mode is specified, we treat all modes in the questionnaireId.
 	 * If no mode is specified and no specs are found, we return an error
-	 * @param campaign campaign id to get modes
+	 * @param questionnaireId questionnaireId id to get modes
 	 * @param modeSpecified a Mode to use, null if we want all modes available
 	 * @return a list with the mode in modeSpecified or all modes if null
-	 * @throws GenesisException if error in specs structure
 	 */
-	public List<Mode> getModesList(String campaign, Mode modeSpecified) throws GenesisException {
+	public List<Mode> getModesList(String questionnaireId, Mode modeSpecified) {
 		if (modeSpecified != null){
 			return Collections.singletonList(modeSpecified);
 		}
 		List<Mode> modes = new ArrayList<>();
-		String specFolder = fileUtils.getSpecFolder(campaign);
+		String specFolder = fileUtils.getSpecFolder(questionnaireId);
 		List<String> modeSpecFolders = fileUtils.listFolders(specFolder);
 		if (modeSpecFolders.isEmpty()) {
-			throw new GenesisException(404, "No specification folder found " + specFolder);
+			throw new UndefinedModesException("No specification folder found " + specFolder);
 		}
 		for(String modeSpecFolder : modeSpecFolders){
 			if(Mode.getEnumFromModeName(modeSpecFolder) == null) {
@@ -51,9 +51,18 @@ public List<Mode> getModesList(String campaign, Mode modeSpecified) throws Genes
 			modes.add(Mode.getEnumFromModeName(modeSpecFolder));
 		}
 		if (modes.contains(Mode.F2F) && modes.contains(Mode.TEL)) {
-			throw new GenesisException(409, "Cannot treat simultaneously TEL and FAF modes");
+			throw new ModesConflictException("Cannot treat simultaneously TEL and FAF modes");
 		}
 		return modes;
 	}
 
+	/**
+	 * Returns the applicable modes for the collection instrument with the given identifier.
+	 * @param collectionInstrumentId Collection instrument identifier.
+	 * @return A list of modes.
+	 */
+	public List<Mode> getModesList(String collectionInstrumentId) {
+		return getModesList(collectionInstrumentId, null);
+	}
+
 }