Skip to content

Commit d39173f

Browse files
Do not use fivetran synced dates older than or equal March 9 2024 [prod]
Signed-off-by: Lukasz Gryglicki <[email protected]> Assisted by [OpenAI](https://platform.openai.com/) Assisted by [GitHub Copilot](https://github.com/features/copilot)
1 parent 12fea18 commit d39173f

File tree

1 file changed

+24
-2
lines changed
  • cla-backend-go/cmd/signatures_timestamp_backfill

1 file changed

+24
-2
lines changed

cla-backend-go/cmd/signatures_timestamp_backfill/main.go

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ const (
3737
attrDateCreated = "date_created"
3838
attrDateModified = "date_modified"
3939

40+
// cutoff date for _FIVETRAN_SYNCED usage
41+
fivetranCutoffDate = "2024-03-09T00:00:00Z"
42+
4043
// update expression helpers
4144
setPrefix = "SET "
4245
commaSep = ", "
@@ -546,8 +549,14 @@ func snowflakeFix(
546549
var newC, srcC string
547550
var newM, srcM string
548551

552+
// Check if _FIVETRAN_SYNCED is before cutoff date
553+
skipFivetran := isFivetranBeforeCutoff(ts)
554+
if skipFivetran {
555+
dbg(" Skipping _FIVETRAN_SYNCED for %s: timestamp %s is on or before cutoff %s", id, ts, fivetranCutoffDate)
556+
}
557+
549558
// CREATED: use _fivetran_synced; clamp to modified if needed
550-
if mC {
559+
if mC && !skipFivetran {
551560
newC, srcC = normalize(ts), labelFivetranSynced
552561
if !isMissing(modified) && after(newC, modified) {
553562
dbg(" SF clamp created: fivetran(%s) > modified(%s) -> modified", newC, modified)
@@ -562,7 +571,7 @@ func snowflakeFix(
562571
newM, srcM = normalize(created), labelFromCreated
563572
case mC && newC != "":
564573
newM, srcM = newC, labelFromCreated
565-
default:
574+
case !skipFivetran:
566575
newM, srcM = normalize(ts), labelFivetranSynced
567576
}
568577
}
@@ -1067,6 +1076,19 @@ func after(a, b string) bool {
10671076
return ta.After(tb)
10681077
}
10691078

1079+
func isFivetranBeforeCutoff(timestamp string) bool {
1080+
ts := normalize(timestamp)
1081+
t, err := time.Parse(time.RFC3339, ts)
1082+
if err != nil {
1083+
return true // if we can't parse, err on the side of caution
1084+
}
1085+
cutoff, err := time.Parse(time.RFC3339, fivetranCutoffDate)
1086+
if err != nil {
1087+
return true // if cutoff is malformed, err on the side of caution
1088+
}
1089+
return !t.After(cutoff) // true if timestamp is on or before cutoff
1090+
}
1091+
10701092
func parseTime(s string) time.Time {
10711093
if strings.TrimSpace(s) == "" {
10721094
return time.Time{}

0 commit comments

Comments
 (0)