diff --git a/apps/go.mod b/apps/go.mod index 908cf21b..9c19acff 100644 --- a/apps/go.mod +++ b/apps/go.mod @@ -24,7 +24,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/lib/pq v1.10.9 github.com/pcelvng/task v0.8.0 - github.com/pcelvng/task-tools v0.29.0 + github.com/pcelvng/task-tools v0.32.0 github.com/prometheus/client_golang v1.18.0 github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.11.1 @@ -41,13 +41,12 @@ require ( github.com/apache/arrow/go/v15 v15.0.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-hostpool v0.1.0 // indirect - github.com/buger/jsonparser v1.1.2 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/google/s2a-go v0.1.9 // indirect diff --git a/apps/go.sum b/apps/go.sum index 4dda9e1f..4b897d43 100644 --- a/apps/go.sum +++ b/apps/go.sum @@ -78,8 +78,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0= github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw= -github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= -github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -303,8 +303,8 @@ github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE= github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY= github.com/pcelvng/task v0.8.0 h1:YA0eXGV801IMt8zePwB15GE126R+pSmyGUeDco3f8dI= github.com/pcelvng/task v0.8.0/go.mod h1:REM+jcZWlxD0b6nSlCow4d51FRLt0y164Ik3sR0Ahag= -github.com/pcelvng/task-tools v0.29.0 h1:Wc7qRCnt3lys/1HGCj1Kp4CcDeGiIk7Jp592yNguFGk= -github.com/pcelvng/task-tools v0.29.0/go.mod h1:v9MDZum0FZr3bhU0HzEUJlCLofUCD5Xio9+OpKMGUHc= +github.com/pcelvng/task-tools v0.32.0 h1:JuP7WHVkQTKMCWcsrikpPyWZ6j4oKBq0eiT5r4bnyos= +github.com/pcelvng/task-tools v0.32.0/go.mod h1:YKkmKpMCFfFdMikPBzwi5kLVjTYvtvLxZrGCZ591YdI= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/dedup/dedup.go b/dedup/dedup.go index df38f733..1264438a 100644 --- a/dedup/dedup.go +++ b/dedup/dedup.go @@ -1,9 +1,10 @@ package dedup import ( - "github.com/buger/jsonparser" "strings" "sync" + + jsoniter "github.com/json-iterator/go" ) func New() *Dedup { @@ -50,8 +51,10 @@ func (w *Dedup) Lines() [][]byte { func KeyFromJSON(b []byte, fields []string) string { var key string for _, field := range fields { - s, _ := jsonparser.GetString(b, field) - key += s + val := jsoniter.Get(b, field) + if val.ValueType() == jsoniter.StringValue { + key += val.ToString() + } } return key } diff --git a/file/writebyhour.go b/file/writebyhour.go index 4c121c71..d5060f2e 100644 --- a/file/writebyhour.go +++ b/file/writebyhour.go @@ -9,7 +9,7 @@ import ( "sync/atomic" "time" - "github.com/buger/jsonparser" + jsoniter "github.com/json-iterator/go" "github.com/pcelvng/task-tools/file/stat" "github.com/pcelvng/task-tools/tmpl" @@ -264,10 +264,13 @@ type jsonDateExtractor struct { // b bytes to be a single json object. func (p *jsonDateExtractor) ExtractDate(b []byte) (time.Time, error) { var t time.Time - s, err := jsonparser.GetString(b, p.fieldName) - if err != nil { + val := jsoniter.Get(b, p.fieldName) + if err := val.LastError(); err != nil { + return t, fmt.Errorf(`field "%v" not in '%v'`, p.fieldName, string(b)) + } + if val.ValueType() != jsoniter.StringValue { return t, fmt.Errorf(`field "%v" not in '%v'`, p.fieldName, string(b)) } - return time.Parse(p.format, s) + return time.Parse(p.format, val.ToString()) } diff --git a/go.mod b/go.mod index 7633bb82..fa6e55f2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/pcelvng/task-tools go 1.24.0 require ( - github.com/buger/jsonparser v1.1.2 github.com/davecgh/go-spew v1.1.1 github.com/go-sql-driver/mysql v1.7.1 github.com/google/go-cmp v0.7.0 @@ -15,6 +14,7 @@ require ( github.com/jbsmith7741/go-tools v0.4.1 github.com/jbsmith7741/uri v0.6.1 github.com/jmoiron/sqlx v1.3.5 + github.com/json-iterator/go v1.1.12 github.com/lib/pq v1.10.9 github.com/minio/minio-go/v7 v7.0.26 github.com/nsqio/go-nsq v1.1.0 @@ -41,7 +41,6 @@ require ( github.com/googleapis/gax-go/v2 v2.14.1 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/iancoleman/strcase v0.3.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid v1.3.1 // indirect diff --git a/go.sum b/go.sum index 0a12d55f..12dfa75c 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0= github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw= -github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= -github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -203,6 +201,7 @@ github.com/jbsmith7741/trial v0.3.1 h1:JZ0/w3lhfH4iacf9R2DnZWtTMa/Uf4O13gnuMLTub github.com/jbsmith7741/trial v0.3.1/go.mod h1:M4FQWUgVpPY2+i53L2nSB0AyPc86kSTIigcr9Q7XQlY= github.com/jbsmith7741/uri v0.4.1/go.mod h1:Ctt8YJ5gCFx5BX/FMFg5VkwuI9buBcvsITIiSMH+TeA= github.com/jbsmith7741/uri v0.6.1 h1:RloJXpTe1lXMBSMbyGn+YCBR//aO+OTDUgJ2wH5bQEU= +github.com/jbsmith7741/uri v0.6.1/go.mod h1:Ctt8YJ5gCFx5BX/FMFg5VkwuI9buBcvsITIiSMH+TeA= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -213,6 +212,7 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= diff --git a/go.work.sum b/go.work.sum index 4e57f05e..d397cece 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1766,6 +1766,7 @@ github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86 h1:D6paGObi5Wu github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c h1:bY6ktFuJkt+ZXkX0RChQch2FtHpWQLVS8Qo1YasiIVk= github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/pcelvng/task-tools v0.29.0/go.mod h1:v9MDZum0FZr3bhU0HzEUJlCLofUCD5Xio9+OpKMGUHc= github.com/phpdave11/gofpdf v1.4.2 h1:KPKiIbfwbvC/wOncwhrpRdXVj2CZTCFlw4wnoyjtHfQ= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=