Skip to content

Commit 5b959f0

Browse files
fix ut
1 parent d58ec31 commit 5b959f0

File tree

2 files changed

+87
-46
lines changed

2 files changed

+87
-46
lines changed

pkg/meta/base.go

Lines changed: 86 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -2904,7 +2904,11 @@ func (m *baseMeta) cloneEntry(ctx Context, srcIno Ino, parent Ino, name string,
29042904
if eno = m.Access(ctx, srcIno, MODE_MASK_R|MODE_MASK_X, &attr); eno != 0 {
29052905
return eno
29062906
}
2907+
// Use DirHandler for batch processing to avoid loading all entries at once
29072908
handler, eno := m.NewDirHandler(ctx, srcIno, false, nil)
2909+
if eno == syscall.ENOENT {
2910+
eno = 0 // empty dir
2911+
}
29082912
if eno != 0 {
29092913
return eno
29102914
}
@@ -2925,70 +2929,107 @@ func (m *baseMeta) cloneEntry(ctx Context, srcIno Ino, parent Ino, name string,
29252929
return eno
29262930
}
29272931

2932+
// Process entries in batches: directories first, then files
2933+
offset := 0
29282934
var dirEntries []*Entry
29292935
var fileEntries []*Entry
2930-
offset := 0
2936+
LOOP:
29312937
for {
2932-
ets, err := handler.List(ctx, offset)
2933-
if err != 0 {
2934-
if err == syscall.ENOENT {
2935-
err = 0 // empty dir
2936-
}
2937-
if err != 0 {
2938-
return err
2938+
// Fetch next batch
2939+
batchEntries, batchEno := handler.List(ctx, offset)
2940+
if batchEno != 0 {
2941+
if batchEno == syscall.ENOENT {
2942+
break // end of directory
29392943
}
2940-
break
2944+
eno = batchEno
2945+
break LOOP
29412946
}
2942-
if len(ets) == 0 {
2943-
break
2947+
if len(batchEntries) == 0 {
2948+
break // no more entries
29442949
}
2945-
for _, e := range ets {
2946-
if len(e.Name) == 1 && e.Name[0] == '.' {
2947-
continue
2948-
}
2949-
if len(e.Name) == 2 && e.Name[0] == '.' && e.Name[1] == '.' {
2950+
2951+
// Separate directories and files in this batch
2952+
actualCount := 0
2953+
for _, e := range batchEntries {
2954+
// Skip "." and ".."
2955+
if len(e.Name) > 0 && (e.Name[0] == '.' && (len(e.Name) == 1 || (len(e.Name) == 2 && e.Name[1] == '.'))) {
29502956
continue
29512957
}
2958+
actualCount++
29522959
if e.Attr.Typ == TypeDirectory {
29532960
dirEntries = append(dirEntries, e)
29542961
} else {
29552962
fileEntries = append(fileEntries, e)
29562963
}
29572964
}
2958-
offset += len(ets)
2959-
if ctx.Canceled() {
2960-
return syscall.EINTR
2961-
}
2962-
}
2963-
2964-
entries := make([]*Entry, 0, len(dirEntries)+len(fileEntries))
2965-
entries = append(entries, dirEntries...)
2966-
entries = append(entries, fileEntries...)
29672965

2968-
LOOP:
2969-
for i, entry := range entries {
2970-
select {
2971-
case e := <-errCh:
2972-
eno = e
2973-
ctx.Cancel()
2974-
break LOOP
2975-
case concurrent <- struct{}{}:
2976-
wg.Add(1)
2977-
go func(e *Entry) {
2978-
defer wg.Done()
2979-
eno := cloneChild(e)
2980-
if eno != 0 {
2981-
errCh <- eno
2966+
// Process directories first
2967+
for _, entry := range dirEntries {
2968+
select {
2969+
case e := <-errCh:
2970+
eno = e
2971+
ctx.Cancel()
2972+
break LOOP
2973+
case concurrent <- struct{}{}:
2974+
wg.Add(1)
2975+
go func(e *Entry) {
2976+
defer wg.Done()
2977+
eno := cloneChild(e)
2978+
if eno != 0 {
2979+
errCh <- eno
2980+
}
2981+
<-concurrent
2982+
}(entry)
2983+
default:
2984+
if e := cloneChild(entry); e != 0 {
2985+
eno = e
2986+
break LOOP
29822987
}
2983-
<-concurrent
2984-
}(entry)
2985-
default:
2986-
if e := cloneChild(entry); e != 0 {
2988+
}
2989+
if ctx.Canceled() {
2990+
eno = syscall.EINTR
2991+
break LOOP
2992+
}
2993+
}
2994+
dirEntries = dirEntries[:0] // clear for next batch
2995+
2996+
// Process files
2997+
for _, entry := range fileEntries {
2998+
select {
2999+
case e := <-errCh:
29873000
eno = e
3001+
ctx.Cancel()
3002+
break LOOP
3003+
case concurrent <- struct{}{}:
3004+
wg.Add(1)
3005+
go func(e *Entry) {
3006+
defer wg.Done()
3007+
eno := cloneChild(e)
3008+
if eno != 0 {
3009+
errCh <- eno
3010+
}
3011+
<-concurrent
3012+
}(entry)
3013+
default:
3014+
if e := cloneChild(entry); e != 0 {
3015+
eno = e
3016+
break LOOP
3017+
}
3018+
}
3019+
if ctx.Canceled() {
3020+
eno = syscall.EINTR
29883021
break LOOP
29893022
}
29903023
}
2991-
entries[i] = nil // release memory
3024+
fileEntries = fileEntries[:0] // clear for next batch
3025+
3026+
// Update offset: List returns entries including initEntries (".", "..")
3027+
// so we need to advance by the total number of entries returned
3028+
offset += len(batchEntries)
3029+
// If we got no actual entries in this batch, we've reached the end
3030+
if actualCount == 0 {
3031+
break
3032+
}
29923033
if ctx.Canceled() {
29933034
eno = syscall.EINTR
29943035
break
@@ -3278,7 +3319,7 @@ func (m *baseMeta) NewDirHandler(ctx Context, inode Ino, plus bool, initEntries
32783319
initEntries = append(initEntries, parent)
32793320

32803321
return m.en.newDirHandler(inode, plus, initEntries), 0
3281-
}
3322+
}
32823323

32833324
type dirBatch struct {
32843325
isEnd bool

pkg/meta/sql_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ func TestPostgreSQLClient(t *testing.T) { //skip mutate
4545
if os.Getenv("SKIP_NON_CORE") == "true" {
4646
t.Skipf("skip non-core test")
4747
}
48-
m, err := newSQLMeta("postgres", "localhost:5432/test?sslmode=disable", testConfig())
48+
m, err := newSQLMeta("postgres", "test:test123@localhost:5432/postgres?sslmode=disable", testConfig())
4949
if err != nil || m.Name() != "postgres" {
5050
t.Fatalf("create meta: %s", err)
5151
}

0 commit comments

Comments
 (0)