Skip to content

Commit daa9a67

Browse files
clostaoclostao
authored
refactor: integrate @autonomys/file-server pkg (#139)
* refactor: integrate pkg * feat: add ACCEPT_UNAUTHORIZED_CERTS environment variable to file-retriever service in docker-compose
1 parent 51d8c7a commit daa9a67

File tree

13 files changed

+85
-179
lines changed

13 files changed

+85
-179
lines changed

‎common/models/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"packageManager": "[email protected]",
55
"type": "module",
66
"dependencies": {
7-
"@autonomys/rpc": "^1.4.31",
7+
"@autonomys/rpc": "^1.5.14",
88
"zod": "^3.24.2"
99
},
1010
"exports": {

‎common/rpc-apis/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"type": "module",
66
"dependencies": {
77
"@auto-files/models": "workspace:*",
8-
"@autonomys/rpc": "^1.5.6",
8+
"@autonomys/rpc": "^1.5.14",
99
"zod": "^3.24.2"
1010
},
1111
"exports": {

‎docker/file-retriever/docker-compose.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ services:
4141
- VICTORIA_PASSWORD=${VICTORIA_PASSWORD}
4242
- OBJECT_MAPPING_INDEXER_URL=http://object-mapping-app:3000/ws
4343
- DATABASE_URL=${DATABASE_URL}
44+
- ACCEPT_UNAUTHORIZED_CERTS=${ACCEPT_UNAUTHORIZED_CERTS}
4445
depends_on:
4546
subspace-node:
4647
condition: service_healthy

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
"test": "yarn workspaces foreach --topological --verbose --all run test"
1919
},
2020
"dependencies": {
21-
"@autonomys/rpc": "^1.4.31",
21+
"@autonomys/rpc": "^1.5.14",
2222
"@subql/types": "^3.14.1",
2323
"axios": "^1.7.9",
2424
"stream-fork": "^1.0.5",

‎services/dag-indexer/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
"typescript": "^5.2.2"
3434
},
3535
"dependencies": {
36-
"@autonomys/auto-dag-data": "^1.4.31",
36+
"@autonomys/auto-dag-data": "^1.5.14",
3737
"@polkadot/util": "^13.5.3",
3838
"@subql/cli": "^6.0.2",
3939
"@subql/common-substrate": "^4.3.7",

‎services/file-retriever/package.json‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,17 @@
44
"type": "module",
55
"dependencies": {
66
"@auto-files/rpc-apis": "workspace:*",
7-
"@autonomys/asynchronous": "^1.5.12",
8-
"@autonomys/auto-dag-data": "^1.5.12",
9-
"@autonomys/auto-utils": "^1.5.12",
10-
"@autonomys/file-caching": "^1.5.12",
11-
"@autonomys/rpc": "^1.5.12",
7+
"@autonomys/asynchronous": "^1.5.14",
8+
"@autonomys/auto-dag-data": "^1.5.14",
9+
"@autonomys/auto-utils": "^1.5.14",
10+
"@autonomys/file-server": "^1.5.14",
11+
"@autonomys/rpc": "^1.5.14",
1212
"@ipld/dag-pb": "^4.1.3",
1313
"@keyvhq/core": "^2.1.1",
1414
"@keyvhq/sqlite": "^2.1.6",
1515
"@polkadot/api": "^15.0.1",
1616
"@polkadot/util": "^13.2.3",
1717
"@types/cors": "^2.8.17",
18-
"@types/express": "^5.0.0",
1918
"@types/mime-types": "^2.1.4",
2019
"@types/node": "^22.10.1",
2120
"cache-manager": "^6.3.0",
@@ -27,7 +26,7 @@
2726
"express-async-errors": "^3.1.1",
2827
"keyv": "^5.2.1",
2928
"lru-cache": "^11.0.2",
30-
"mime-types": "^2.1.35",
29+
"mime-types": "^3.0.1",
3130
"p-limit": "^6.2.0",
3231
"typescript": "^5.7.2"
3332
},
@@ -41,6 +40,7 @@
4140
},
4241
"devDependencies": {
4342
"@types/cpu-features": "^0",
43+
"@types/express": "^5.0.3",
4444
"@types/jest": "^29.5.14",
4545
"@types/pg": "^8.11.10",
4646
"@types/pg-format": "^1",

‎services/file-retriever/src/http/controllers/file.ts‎

Lines changed: 17 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,16 @@ import { fileComposer } from '../../services/fileComposer.js'
44
import { pipeline } from 'stream'
55
import { logger } from '../../drivers/logger.js'
66
import { asyncSafeHandler } from '../../utils/express.js'
7-
import { getByteRange, uniqueHeaderValue } from '../../utils/http.js'
7+
import { uniqueHeaderValue } from '../../utils/http.js'
88
import { HttpError } from '../middlewares/error.js'
99
import { dsnFetcher } from '../../services/dsnFetcher.js'
1010
import { isValidCID } from '../../utils/dagData.js'
1111
import { fileCache } from '../../services/cache.js'
12+
import {
13+
DownloadMetadataFactory,
14+
handleDownloadResponseHeaders,
15+
getByteRange,
16+
} from '@autonomys/file-server'
1217

1318
const fileRouter = Router()
1419

@@ -75,47 +80,17 @@ fileRouter.get(
7580
ignoreCache,
7681
byteRange,
7782
})
78-
if (fromCache) {
79-
res.setHeader('x-file-origin', 'cache')
80-
} else {
81-
res.setHeader('x-file-origin', 'gateway')
82-
}
83-
84-
if (file.mimeType) {
85-
res.set('Content-Type', file.mimeType)
86-
}
87-
if (file.filename) {
88-
res.set(
89-
'Content-Disposition',
90-
`filename="${encodeURIComponent(file.filename)}"`,
91-
)
92-
}
93-
94-
// Advertise range support
95-
res.set('Accept-Ranges', 'bytes')
96-
97-
if (byteRange && file.size != null) {
98-
const fileSizeNumber = Number(file.size)
99-
const startIndex = byteRange[0]
100-
101-
const effectiveEnd = Math.min(
102-
byteRange[1] != null ? byteRange[1] : fileSizeNumber - 1,
103-
fileSizeNumber - 1,
104-
)
105-
const contentLength = effectiveEnd - startIndex + 1
106-
107-
res.status(206)
108-
res.set(
109-
'Content-Range',
110-
`bytes ${startIndex}-${effectiveEnd}/${fileSizeNumber}`,
111-
)
112-
res.set('Content-Length', contentLength.toString())
113-
} else if (file.size != null) {
114-
res.set('Content-Length', file.size.toString())
115-
}
116-
if (file.encoding && !rawMode && !byteRange) {
117-
res.set('Content-Encoding', file.encoding)
118-
}
83+
res.setHeader('x-file-origin', fromCache ? 'cache' : 'gateway')
84+
85+
handleDownloadResponseHeaders(
86+
req,
87+
res,
88+
DownloadMetadataFactory.fromIPLDData(metadata),
89+
{
90+
byteRange,
91+
rawMode,
92+
},
93+
)
11994

12095
logger.debug(
12196
`Streaming file ${req.params.cid} to ${req.ip} with ${file.size} bytes`,

‎services/file-retriever/src/services/cache.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import {
44
ensureDirectoryExists,
55
FileCacheOptions,
66
FileResponse,
7-
} from '@autonomys/file-caching'
7+
} from '@autonomys/file-server'
88
import path from 'path'
99
import { config } from '../config.js'
1010

‎services/file-retriever/src/services/dsnFetcher.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import {
1313
ByteRange,
1414
FileCacheOptions,
1515
FileResponse,
16-
} from '@autonomys/file-caching'
16+
} from '@autonomys/file-server'
1717
import { z } from 'zod'
1818
import { PBNode } from '@ipld/dag-pb'
1919
import { HttpError } from '../http/middlewares/error.js'
@@ -529,4 +529,5 @@ export const dsnFetcher = {
529529
getPartial,
530530
fetchNodeMetadata,
531531
getFileChunks,
532+
getFileMetadata,
532533
}

‎services/file-retriever/src/services/fileComposer.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { dsnFetcher } from './dsnFetcher.js'
22
import { forkStream } from '@autonomys/asynchronous'
33
import { logger } from '../drivers/logger.js'
4-
import { FileCacheOptions, FileResponse } from '@autonomys/file-caching'
4+
import { FileCacheOptions, FileResponse } from '@autonomys/file-server'
55
import { fileCache } from './cache.js'
66
import { moderationService } from './moderation.js'
77
import { HttpError } from '../http/middlewares/error.js'

0 commit comments

Comments
 (0)