From 9f441873516490c84bcc64615cce1ae1a84b6189 Mon Sep 17 00:00:00 2001 From: Sebastien Bruel <93573440+sbruel@users.noreply.github.com> Date: Sat, 12 Jul 2025 05:51:53 +0900 Subject: [PATCH] =?UTF-8?q?=F0=9F=97=82=EF=B8=8F=20fix:=20Disable=20`expre?= =?UTF-8?q?ss-static-gzip`=20for=20Uploaded=20Images=20(#8307)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix scanning of the uploaded images folder on startup * Re-write tests to pass linting * Disable image output gzip scan by default * Add `ENABLE_IMAGE_OUTPUT_GZIP_SCAN` to `.env.example` --- .env.example | 4 + api/server/routes/__tests__/static.spec.js | 162 +++++++ api/server/routes/static.js | 5 +- .../utils/__tests__/staticCache.spec.js | 407 ++++++++++++++++++ api/server/utils/staticCache.js | 74 ++-- 5 files changed, 620 insertions(+), 32 deletions(-) create mode 100644 api/server/routes/__tests__/static.spec.js create mode 100644 api/server/utils/__tests__/staticCache.spec.js diff --git a/.env.example b/.env.example index 2745881992..c39ad41250 100644 --- a/.env.example +++ b/.env.example @@ -580,6 +580,10 @@ ALLOW_SHARED_LINKS_PUBLIC=true # If you have another service in front of your LibreChat doing compression, disable express based compression here # DISABLE_COMPRESSION=true +# If you have gzipped version of uploaded image images in the same folder, this will enable gzip scan and serving of these images +# Note: The images folder will be scanned on startup and a ma kept in memory. Be careful for large number of images. +# ENABLE_IMAGE_OUTPUT_GZIP_SCAN=true + #===================================================# # UI # #===================================================# diff --git a/api/server/routes/__tests__/static.spec.js b/api/server/routes/__tests__/static.spec.js new file mode 100644 index 0000000000..239ad7e095 --- /dev/null +++ b/api/server/routes/__tests__/static.spec.js @@ -0,0 +1,162 @@ +const fs = require('fs'); +const path = require('path'); +const express = require('express'); +const request = require('supertest'); +const zlib = require('zlib'); + +// Create test setup +const mockTestDir = path.join(__dirname, 'test-static-route'); + +// Mock the paths module to point to our test directory +jest.mock('~/config/paths', () => ({ + imageOutput: mockTestDir, +})); + +describe('Static Route Integration', () => { + let app; + let staticRoute; + let testDir; + let testImagePath; + + beforeAll(() => { + // Create a test directory and files + testDir = mockTestDir; + testImagePath = path.join(testDir, 'test-image.jpg'); + + if (!fs.existsSync(testDir)) { + fs.mkdirSync(testDir, { recursive: true }); + } + + // Create a test image file + fs.writeFileSync(testImagePath, 'fake-image-data'); + + // Create a gzipped version of the test image (for gzip scanning tests) + fs.writeFileSync(testImagePath + '.gz', zlib.gzipSync('fake-image-data')); + }); + + afterAll(() => { + // Clean up test files + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true, force: true }); + } + }); + + // Helper function to set up static route with specific config + const setupStaticRoute = (skipGzipScan = false) => { + if (skipGzipScan) { + delete process.env.ENABLE_IMAGE_OUTPUT_GZIP_SCAN; + } else { + process.env.ENABLE_IMAGE_OUTPUT_GZIP_SCAN = 'true'; + } + + staticRoute = require('../static'); + app.use('/images', staticRoute); + }; + + beforeEach(() => { + // Clear the module cache to get fresh imports + jest.resetModules(); + + app = express(); + + // Clear environment variables + delete process.env.ENABLE_IMAGE_OUTPUT_GZIP_SCAN; + delete process.env.NODE_ENV; + }); + + describe('route functionality', () => { + it('should serve static image files', async () => { + process.env.NODE_ENV = 'production'; + setupStaticRoute(); + + const response = await request(app).get('/images/test-image.jpg').expect(200); + + expect(response.body.toString()).toBe('fake-image-data'); + }); + + it('should return 404 for non-existent files', async () => { + setupStaticRoute(); + + const response = await request(app).get('/images/nonexistent.jpg'); + expect(response.status).toBe(404); + }); + }); + + describe('cache behavior', () => { + it('should set cache headers for images in production', async () => { + process.env.NODE_ENV = 'production'; + setupStaticRoute(); + + const response = await request(app).get('/images/test-image.jpg').expect(200); + + expect(response.headers['cache-control']).toBe('public, max-age=172800, s-maxage=86400'); + }); + + it('should not set cache headers in development', async () => { + process.env.NODE_ENV = 'development'; + setupStaticRoute(); + + const response = await request(app).get('/images/test-image.jpg').expect(200); + + // Our middleware should not set the production cache-control header in development + expect(response.headers['cache-control']).not.toBe('public, max-age=172800, s-maxage=86400'); + }); + }); + + describe('gzip compression behavior', () => { + beforeEach(() => { + process.env.NODE_ENV = 'production'; + }); + + it('should serve gzipped files when gzip scanning is enabled', async () => { + setupStaticRoute(false); // Enable gzip scanning + + const response = await request(app) + .get('/images/test-image.jpg') + .set('Accept-Encoding', 'gzip') + .expect(200); + + expect(response.headers['content-encoding']).toBe('gzip'); + expect(response.body.toString()).toBe('fake-image-data'); + }); + + it('should not serve gzipped files when gzip scanning is disabled', async () => { + setupStaticRoute(true); // Disable gzip scanning + + const response = await request(app) + .get('/images/test-image.jpg') + .set('Accept-Encoding', 'gzip') + .expect(200); + + expect(response.headers['content-encoding']).toBeUndefined(); + expect(response.body.toString()).toBe('fake-image-data'); + }); + }); + + describe('path configuration', () => { + it('should use the configured imageOutput path', async () => { + setupStaticRoute(); + + const response = await request(app).get('/images/test-image.jpg').expect(200); + + expect(response.body.toString()).toBe('fake-image-data'); + }); + + it('should serve from subdirectories', async () => { + // Create a subdirectory with a file + const subDir = path.join(testDir, 'thumbs'); + fs.mkdirSync(subDir, { recursive: true }); + const thumbPath = path.join(subDir, 'thumb.jpg'); + fs.writeFileSync(thumbPath, 'thumbnail-data'); + + setupStaticRoute(); + + const response = await request(app).get('/images/thumbs/thumb.jpg').expect(200); + + expect(response.body.toString()).toBe('thumbnail-data'); + + // Clean up + fs.rmSync(subDir, { recursive: true, force: true }); + }); + }); +}); diff --git a/api/server/routes/static.js b/api/server/routes/static.js index 2db55ebebc..952ca82fb2 100644 --- a/api/server/routes/static.js +++ b/api/server/routes/static.js @@ -1,8 +1,11 @@ const express = require('express'); const staticCache = require('../utils/staticCache'); const paths = require('~/config/paths'); +const { isEnabled } = require('~/server/utils'); + +const skipGzipScan = !isEnabled(process.env.ENABLE_IMAGE_OUTPUT_GZIP_SCAN); const router = express.Router(); -router.use(staticCache(paths.imageOutput)); +router.use(staticCache(paths.imageOutput, { skipGzipScan })); module.exports = router; diff --git a/api/server/utils/__tests__/staticCache.spec.js b/api/server/utils/__tests__/staticCache.spec.js new file mode 100644 index 0000000000..5d285017bd --- /dev/null +++ b/api/server/utils/__tests__/staticCache.spec.js @@ -0,0 +1,407 @@ +const fs = require('fs'); +const path = require('path'); +const express = require('express'); +const request = require('supertest'); +const zlib = require('zlib'); +const staticCache = require('../staticCache'); + +describe('staticCache', () => { + let app; + let testDir; + let testFile; + let indexFile; + let manifestFile; + let swFile; + + beforeAll(() => { + // Create a test directory and files + testDir = path.join(__dirname, 'test-static'); + if (!fs.existsSync(testDir)) { + fs.mkdirSync(testDir, { recursive: true }); + } + + // Create test files + testFile = path.join(testDir, 'test.js'); + indexFile = path.join(testDir, 'index.html'); + manifestFile = path.join(testDir, 'manifest.json'); + swFile = path.join(testDir, 'sw.js'); + + const jsContent = 'console.log("test");'; + const htmlContent = 'Test'; + const jsonContent = '{"name": "test"}'; + const swContent = 'self.addEventListener("install", () => {});'; + + fs.writeFileSync(testFile, jsContent); + fs.writeFileSync(indexFile, htmlContent); + fs.writeFileSync(manifestFile, jsonContent); + fs.writeFileSync(swFile, swContent); + + // Create gzipped versions of some files + fs.writeFileSync(testFile + '.gz', zlib.gzipSync(jsContent)); + fs.writeFileSync(path.join(testDir, 'test.css'), 'body { color: red; }'); + fs.writeFileSync(path.join(testDir, 'test.css.gz'), zlib.gzipSync('body { color: red; }')); + + // Create a file that only exists in gzipped form + fs.writeFileSync( + path.join(testDir, 'only-gzipped.js.gz'), + zlib.gzipSync('console.log("only gzipped");'), + ); + + // Create a subdirectory for dist/images testing + const distImagesDir = path.join(testDir, 'dist', 'images'); + fs.mkdirSync(distImagesDir, { recursive: true }); + fs.writeFileSync(path.join(distImagesDir, 'logo.png'), 'fake-png-data'); + }); + + afterAll(() => { + // Clean up test files + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true, force: true }); + } + }); + + beforeEach(() => { + app = express(); + + // Clear environment variables + delete process.env.NODE_ENV; + delete process.env.STATIC_CACHE_S_MAX_AGE; + delete process.env.STATIC_CACHE_MAX_AGE; + }); + describe('cache headers in production', () => { + beforeEach(() => { + process.env.NODE_ENV = 'production'; + }); + + it('should set standard cache headers for regular files', async () => { + app.use(staticCache(testDir)); + + const response = await request(app).get('/test.js').expect(200); + + expect(response.headers['cache-control']).toBe('public, max-age=172800, s-maxage=86400'); + }); + + it('should set no-cache headers for index.html', async () => { + app.use(staticCache(testDir)); + + const response = await request(app).get('/index.html').expect(200); + + expect(response.headers['cache-control']).toBe('no-store, no-cache, must-revalidate'); + }); + + it('should set no-cache headers for manifest.json', async () => { + app.use(staticCache(testDir)); + + const response = await request(app).get('/manifest.json').expect(200); + + expect(response.headers['cache-control']).toBe('no-store, no-cache, must-revalidate'); + }); + + it('should set no-cache headers for sw.js', async () => { + app.use(staticCache(testDir)); + + const response = await request(app).get('/sw.js').expect(200); + + expect(response.headers['cache-control']).toBe('no-store, no-cache, must-revalidate'); + }); + + it('should not set cache headers for /dist/images/ files', async () => { + app.use(staticCache(testDir)); + + const response = await request(app).get('/dist/images/logo.png').expect(200); + + expect(response.headers['cache-control']).toBe('public, max-age=0'); + }); + + it('should set no-cache headers when noCache option is true', async () => { + app.use(staticCache(testDir, { noCache: true })); + + const response = await request(app).get('/test.js').expect(200); + + expect(response.headers['cache-control']).toBe('no-store, no-cache, must-revalidate'); + }); + }); + + describe('cache headers in non-production', () => { + beforeEach(() => { + process.env.NODE_ENV = 'development'; + }); + + it('should not set cache headers in development', async () => { + app.use(staticCache(testDir)); + + const response = await request(app).get('/test.js').expect(200); + + // Our middleware should not set cache-control in non-production + // Express static might set its own default headers + const cacheControl = response.headers['cache-control']; + expect(cacheControl).toBe('public, max-age=0'); + }); + }); + + describe('environment variable configuration', () => { + beforeEach(() => { + process.env.NODE_ENV = 'production'; + }); + + it('should use custom s-maxage from environment', async () => { + process.env.STATIC_CACHE_S_MAX_AGE = '3600'; + + // Need to re-require to pick up new env vars + jest.resetModules(); + const freshStaticCache = require('../staticCache'); + + app.use(freshStaticCache(testDir)); + + const response = await request(app).get('/test.js').expect(200); + + expect(response.headers['cache-control']).toBe('public, max-age=172800, s-maxage=3600'); + }); + + it('should use custom max-age from environment', async () => { + process.env.STATIC_CACHE_MAX_AGE = '7200'; + + // Need to re-require to pick up new env vars + jest.resetModules(); + const freshStaticCache = require('../staticCache'); + + app.use(freshStaticCache(testDir)); + + const response = await request(app).get('/test.js').expect(200); + + expect(response.headers['cache-control']).toBe('public, max-age=7200, s-maxage=86400'); + }); + + it('should use both custom values from environment', async () => { + process.env.STATIC_CACHE_S_MAX_AGE = '1800'; + process.env.STATIC_CACHE_MAX_AGE = '3600'; + + // Need to re-require to pick up new env vars + jest.resetModules(); + const freshStaticCache = require('../staticCache'); + + app.use(freshStaticCache(testDir)); + + const response = await request(app).get('/test.js').expect(200); + + expect(response.headers['cache-control']).toBe('public, max-age=3600, s-maxage=1800'); + }); + }); + + describe('express-static-gzip behavior', () => { + beforeEach(() => { + process.env.NODE_ENV = 'production'; + }); + + it('should serve gzipped files when client accepts gzip encoding', async () => { + app.use(staticCache(testDir, { skipGzipScan: false })); + + const response = await request(app) + .get('/test.js') + .set('Accept-Encoding', 'gzip, deflate') + .expect(200); + + expect(response.headers['content-encoding']).toBe('gzip'); + expect(response.headers['content-type']).toMatch(/javascript/); + expect(response.headers['cache-control']).toBe('public, max-age=172800, s-maxage=86400'); + // Content should be decompressed by supertest + expect(response.text).toBe('console.log("test");'); + }); + + it('should fall back to uncompressed files when client does not accept gzip', async () => { + app.use(staticCache(testDir, { skipGzipScan: false })); + + const response = await request(app) + .get('/test.js') + .set('Accept-Encoding', 'identity') + .expect(200); + + expect(response.headers['content-encoding']).toBeUndefined(); + expect(response.headers['content-type']).toMatch(/javascript/); + expect(response.text).toBe('console.log("test");'); + }); + + it('should serve gzipped CSS files with correct content-type', async () => { + app.use(staticCache(testDir, { skipGzipScan: false })); + + const response = await request(app) + .get('/test.css') + .set('Accept-Encoding', 'gzip') + .expect(200); + + expect(response.headers['content-encoding']).toBe('gzip'); + expect(response.headers['content-type']).toMatch(/css/); + expect(response.text).toBe('body { color: red; }'); + }); + + it('should serve uncompressed files when no gzipped version exists', async () => { + app.use(staticCache(testDir, { skipGzipScan: false })); + + const response = await request(app) + .get('/manifest.json') + .set('Accept-Encoding', 'gzip') + .expect(200); + + expect(response.headers['content-encoding']).toBeUndefined(); + expect(response.headers['content-type']).toMatch(/json/); + expect(response.text).toBe('{"name": "test"}'); + }); + + it('should handle files that only exist in gzipped form', async () => { + app.use(staticCache(testDir, { skipGzipScan: false })); + + const response = await request(app) + .get('/only-gzipped.js') + .set('Accept-Encoding', 'gzip') + .expect(200); + + expect(response.headers['content-encoding']).toBe('gzip'); + expect(response.headers['content-type']).toMatch(/javascript/); + expect(response.text).toBe('console.log("only gzipped");'); + }); + + it('should return 404 for gzip-only files when client does not accept gzip', async () => { + app.use(staticCache(testDir, { skipGzipScan: false })); + + const response = await request(app) + .get('/only-gzipped.js') + .set('Accept-Encoding', 'identity'); + expect(response.status).toBe(404); + }); + + it('should handle cache headers correctly for gzipped content', async () => { + app.use(staticCache(testDir, { skipGzipScan: false })); + + const response = await request(app) + .get('/test.js') + .set('Accept-Encoding', 'gzip') + .expect(200); + + expect(response.headers['content-encoding']).toBe('gzip'); + expect(response.headers['cache-control']).toBe('public, max-age=172800, s-maxage=86400'); + expect(response.headers['content-type']).toMatch(/javascript/); + }); + + it('should preserve original MIME types for gzipped files', async () => { + app.use(staticCache(testDir, { skipGzipScan: false })); + + const jsResponse = await request(app) + .get('/test.js') + .set('Accept-Encoding', 'gzip') + .expect(200); + + const cssResponse = await request(app) + .get('/test.css') + .set('Accept-Encoding', 'gzip') + .expect(200); + + expect(jsResponse.headers['content-type']).toMatch(/javascript/); + expect(cssResponse.headers['content-type']).toMatch(/css/); + expect(jsResponse.headers['content-encoding']).toBe('gzip'); + expect(cssResponse.headers['content-encoding']).toBe('gzip'); + }); + }); + + describe('skipGzipScan option comparison', () => { + beforeEach(() => { + process.env.NODE_ENV = 'production'; + }); + + it('should use express.static (no gzip) when skipGzipScan is true', async () => { + app.use(staticCache(testDir, { skipGzipScan: true })); + + const response = await request(app) + .get('/test.js') + .set('Accept-Encoding', 'gzip') + .expect(200); + + // Should NOT serve gzipped version even though client accepts it + expect(response.headers['content-encoding']).toBeUndefined(); + expect(response.headers['cache-control']).toBe('public, max-age=172800, s-maxage=86400'); + expect(response.text).toBe('console.log("test");'); + }); + + it('should use expressStaticGzip when skipGzipScan is false', async () => { + app.use(staticCache(testDir)); + + const response = await request(app) + .get('/test.js') + .set('Accept-Encoding', 'gzip') + .expect(200); + + // Should serve gzipped version when client accepts it + expect(response.headers['content-encoding']).toBe('gzip'); + expect(response.headers['cache-control']).toBe('public, max-age=172800, s-maxage=86400'); + expect(response.text).toBe('console.log("test");'); + }); + }); + + describe('file serving', () => { + beforeEach(() => { + process.env.NODE_ENV = 'production'; + }); + + it('should serve files correctly', async () => { + app.use(staticCache(testDir)); + + const response = await request(app).get('/test.js').expect(200); + + expect(response.text).toBe('console.log("test");'); + expect(response.headers['content-type']).toMatch(/javascript|text/); + }); + + it('should return 404 for non-existent files', async () => { + app.use(staticCache(testDir)); + + const response = await request(app).get('/nonexistent.js'); + expect(response.status).toBe(404); + }); + + it('should serve HTML files', async () => { + app.use(staticCache(testDir)); + + const response = await request(app).get('/index.html').expect(200); + + expect(response.text).toBe('Test'); + expect(response.headers['content-type']).toMatch(/html/); + }); + }); + + describe('edge cases', () => { + beforeEach(() => { + process.env.NODE_ENV = 'production'; + }); + + it('should handle webmanifest files', async () => { + // Create a webmanifest file + const webmanifestFile = path.join(testDir, 'site.webmanifest'); + fs.writeFileSync(webmanifestFile, '{"name": "test app"}'); + + app.use(staticCache(testDir)); + + const response = await request(app).get('/site.webmanifest').expect(200); + + expect(response.headers['cache-control']).toBe('no-store, no-cache, must-revalidate'); + + // Clean up + fs.unlinkSync(webmanifestFile); + }); + + it('should handle files in subdirectories', async () => { + const subDir = path.join(testDir, 'subdir'); + fs.mkdirSync(subDir, { recursive: true }); + const subFile = path.join(subDir, 'nested.js'); + fs.writeFileSync(subFile, 'console.log("nested");'); + + app.use(staticCache(testDir)); + + const response = await request(app).get('/subdir/nested.js').expect(200); + + expect(response.headers['cache-control']).toBe('public, max-age=172800, s-maxage=86400'); + expect(response.text).toBe('console.log("nested");'); + + // Clean up + fs.rmSync(subDir, { recursive: true, force: true }); + }); + }); +}); diff --git a/api/server/utils/staticCache.js b/api/server/utils/staticCache.js index e885273223..ecaea856d0 100644 --- a/api/server/utils/staticCache.js +++ b/api/server/utils/staticCache.js @@ -1,4 +1,5 @@ const path = require('path'); +const express = require('express'); const expressStaticGzip = require('express-static-gzip'); const oneDayInSeconds = 24 * 60 * 60; @@ -7,44 +8,55 @@ const sMaxAge = process.env.STATIC_CACHE_S_MAX_AGE || oneDayInSeconds; const maxAge = process.env.STATIC_CACHE_MAX_AGE || oneDayInSeconds * 2; /** - * Creates an Express static middleware with gzip compression and configurable caching + * Creates an Express static middleware with optional gzip compression and configurable caching * * @param {string} staticPath - The file system path to serve static files from * @param {Object} [options={}] - Configuration options * @param {boolean} [options.noCache=false] - If true, disables caching entirely for all files - * @returns {ReturnType} Express middleware function for serving static files + * @param {boolean} [options.skipGzipScan=false] - If true, skips expressStaticGzip middleware + * @returns {ReturnType|ReturnType} Express middleware function for serving static files */ function staticCache(staticPath, options = {}) { - const { noCache = false } = options; - return expressStaticGzip(staticPath, { - enableBrotli: false, - orderPreference: ['gz'], - setHeaders: (res, filePath) => { - if (process.env.NODE_ENV?.toLowerCase() !== 'production') { - return; - } - if (noCache) { - res.setHeader('Cache-Control', 'no-store, no-cache, must-revalidate'); - return; - } - if (filePath.includes('/dist/images/')) { - return; - } - const fileName = path.basename(filePath); + const { noCache = false, skipGzipScan = false } = options; - if ( - fileName === 'index.html' || - fileName.endsWith('.webmanifest') || - fileName === 'manifest.json' || - fileName === 'sw.js' - ) { - res.setHeader('Cache-Control', 'no-store, no-cache, must-revalidate'); - } else { - res.setHeader('Cache-Control', `public, max-age=${maxAge}, s-maxage=${sMaxAge}`); - } - }, - index: false, - }); + const setHeaders = (res, filePath) => { + if (process.env.NODE_ENV?.toLowerCase() !== 'production') { + return; + } + if (noCache) { + res.setHeader('Cache-Control', 'no-store, no-cache, must-revalidate'); + return; + } + if (filePath && filePath.includes('/dist/images/')) { + return; + } + const fileName = filePath ? path.basename(filePath) : ''; + + if ( + fileName === 'index.html' || + fileName.endsWith('.webmanifest') || + fileName === 'manifest.json' || + fileName === 'sw.js' + ) { + res.setHeader('Cache-Control', 'no-store, no-cache, must-revalidate'); + } else { + res.setHeader('Cache-Control', `public, max-age=${maxAge}, s-maxage=${sMaxAge}`); + } + }; + + if (skipGzipScan) { + return express.static(staticPath, { + setHeaders, + index: false, + }); + } else { + return expressStaticGzip(staticPath, { + enableBrotli: false, + orderPreference: ['gz'], + setHeaders, + index: false, + }); + } } module.exports = staticCache;