From 6536aa31997b72d84929eb2c4de2ca2edf51d540 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 22 Nov 2025 03:27:11 +0000 Subject: [PATCH] feat: add file validation with magic byte checking Add comprehensive file validation to prevent malicious file uploads: - Implement magic byte (file signature) validation for all supported file types - Validate file extensions match declared content types - Support all image formats: PNG, JPEG, WebP, SVG, TIFF - Support all video formats: MP4, MPEG, OGG - Support document formats: PDF, TXT - Integrate validation into upload controller before processing - Add comprehensive test suite with 40+ test cases Security improvements: - Prevent upload of files with fake extensions (e.g., malware.jpg that's actually .exe) - Verify actual file content matches Content-Type header - Detect tampered or corrupted file headers - Early rejection of invalid files before AWS upload The validator checks magic bytes at correct offsets: - Images: PNG (8 bytes), JPEG (4 bytes), WebP (RIFF+WEBP), TIFF (4 bytes) - Videos: MP4 (offset 4), MPEG (4 bytes), OGG (4 bytes) - Documents: PDF (%PDF marker) - SVG: XML-based validation Closes security gap identified in code review. --- internal/controllers/upload_controller.go | 6 + pkg/utils/file_validator.go | 237 ++++++++++++++ pkg/utils/file_validator_test.go | 359 ++++++++++++++++++++++ 3 files changed, 602 insertions(+) create mode 100644 pkg/utils/file_validator.go create mode 100644 pkg/utils/file_validator_test.go diff --git a/internal/controllers/upload_controller.go b/internal/controllers/upload_controller.go index 3dd37d6..e1521da 100644 --- a/internal/controllers/upload_controller.go +++ b/internal/controllers/upload_controller.go @@ -100,6 +100,12 @@ func (u *UploadController) Upload(c *gin.Context) { return } + // Validate file signature (magic bytes) matches content type + if err := utils.ValidateFile(fileHeader, contentType); err != nil { + abortWithBadRequest(c, "file validation failed", err.Error()) + return + } + eventType, uploader, err := uploader.GetUploaderByContentType(contentType) if err != nil { abortWithBadRequest(c, "error validating file content type", err.Error()) diff --git a/pkg/utils/file_validator.go b/pkg/utils/file_validator.go new file mode 100644 index 0000000..16cb461 --- /dev/null +++ b/pkg/utils/file_validator.go @@ -0,0 +1,237 @@ +package utils + +import ( + "bytes" + "errors" + "fmt" + "mime/multipart" +) + +var ( + // ErrInvalidFileSignature is returned when file magic bytes don't match content type + ErrInvalidFileSignature = errors.New("file signature does not match content type") + + // ErrUnsupportedFileType is returned when file type is not supported + ErrUnsupportedFileType = errors.New("unsupported file type") +) + +// FileSignature represents a file's magic byte signature +type FileSignature struct { + ContentType string + Signatures [][]byte + Offset int // offset where signature starts +} + +// Common file signatures (magic bytes) +var fileSignatures = []FileSignature{ + // Image formats + { + ContentType: "image/png", + Signatures: [][]byte{ + {0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, // PNG signature + }, + Offset: 0, + }, + { + ContentType: "image/jpeg", + Signatures: [][]byte{ + {0xFF, 0xD8, 0xFF, 0xDB}, // JPEG raw + {0xFF, 0xD8, 0xFF, 0xE0}, // JPEG/JFIF + {0xFF, 0xD8, 0xFF, 0xE1}, // JPEG/Exif + {0xFF, 0xD8, 0xFF, 0xE2}, // JPEG/Canon + {0xFF, 0xD8, 0xFF, 0xE3}, // JPEG/Samsung + {0xFF, 0xD8, 0xFF, 0xEE}, // JPEG/Adobe + }, + Offset: 0, + }, + { + ContentType: "image/jpg", // Alias for JPEG + Signatures: [][]byte{ + {0xFF, 0xD8, 0xFF, 0xDB}, + {0xFF, 0xD8, 0xFF, 0xE0}, + {0xFF, 0xD8, 0xFF, 0xE1}, + {0xFF, 0xD8, 0xFF, 0xE2}, + {0xFF, 0xD8, 0xFF, 0xE3}, + {0xFF, 0xD8, 0xFF, 0xEE}, + }, + Offset: 0, + }, + { + ContentType: "image/webp", + Signatures: [][]byte{ + {0x52, 0x49, 0x46, 0x46}, // RIFF (WebP starts with RIFF) + }, + Offset: 0, + }, + { + ContentType: "image/tiff", + Signatures: [][]byte{ + {0x49, 0x49, 0x2A, 0x00}, // Little-endian TIFF + {0x4D, 0x4D, 0x00, 0x2A}, // Big-endian TIFF + }, + Offset: 0, + }, + { + ContentType: "image/svg+xml", + Signatures: [][]byte{ + {0x3C, 0x3F, 0x78, 0x6D, 0x6C}, // len(buffer) { + continue + } + + fileBytes := buffer[matchingSignature.Offset : matchingSignature.Offset+sigLen] + if bytes.Equal(fileBytes, signature) { + // Additional check for WebP - verify WEBP string at offset 8 + if contentType == "image/webp" { + if len(buffer) >= 12 { + webpMarker := buffer[8:12] + if bytes.Equal(webpMarker, []byte{0x57, 0x45, 0x42, 0x50}) { + return nil + } + } + continue // Try next signature if WEBP marker not found + } + return nil // Valid signature found + } + } + + return fmt.Errorf("%w: expected %s but file signature doesn't match", + ErrInvalidFileSignature, contentType) +} + +// ValidateFileExtension checks if file extension matches content type +func ValidateFileExtension(filename string, contentType string) error { + expectedExtensions := map[string][]string{ + "image/png": {".png"}, + "image/jpeg": {".jpg", ".jpeg"}, + "image/jpg": {".jpg", ".jpeg"}, + "image/svg+xml": {".svg"}, + "image/webp": {".webp"}, + "image/tiff": {".tiff", ".tif"}, + "video/mp4": {".mp4"}, + "video/mpeg": {".mpeg", ".mpg"}, + "video/ogg": {".ogg", ".ogv"}, + "application/pdf": {".pdf"}, + "text/plain": {".txt"}, + } + + extensions, ok := expectedExtensions[contentType] + if !ok { + return fmt.Errorf("%w: %s", ErrUnsupportedFileType, contentType) + } + + // Get file extension (case-insensitive) + for _, ext := range extensions { + if len(filename) >= len(ext) { + fileExt := filename[len(filename)-len(ext):] + if bytes.EqualFold([]byte(fileExt), []byte(ext)) { + return nil + } + } + } + + return fmt.Errorf("file extension doesn't match content type %s (expected one of %v)", + contentType, extensions) +} + +// ValidateFile performs comprehensive file validation +func ValidateFile(fileHeader *multipart.FileHeader, contentType string) error { + // 1. Validate file extension + if err := ValidateFileExtension(fileHeader.Filename, contentType); err != nil { + return err + } + + // 2. Validate magic bytes + if err := ValidateFileSignature(fileHeader, contentType); err != nil { + return err + } + + return nil +} diff --git a/pkg/utils/file_validator_test.go b/pkg/utils/file_validator_test.go new file mode 100644 index 0000000..f05a361 --- /dev/null +++ b/pkg/utils/file_validator_test.go @@ -0,0 +1,359 @@ +package utils + +import ( + "bytes" + "io" + "mime/multipart" + "net/textproto" + "testing" + + "github.com/stretchr/testify/assert" +) + +// createMockFileHeader creates a mock multipart.FileHeader for testing +func createMockFileHeader(filename string, content []byte) *multipart.FileHeader { + header := &multipart.FileHeader{ + Filename: filename, + Size: int64(len(content)), + Header: make(textproto.MIMEHeader), + } + + // Create a pipe to simulate file content + pr, pw := io.Pipe() + writer := multipart.NewWriter(pw) + + go func() { + defer pw.Close() + part, _ := writer.CreateFormFile("file", filename) + part.Write(content) + writer.Close() + }() + + reader := multipart.NewReader(pr, writer.Boundary()) + part, _ := reader.NextPart() + + // Read the content back + buf := new(bytes.Buffer) + buf.ReadFrom(part) + + // Store content for Open() method + header.Header = part.Header + + return header +} + +// createTestFileHeader creates a proper test file header with content +func createTestFileHeader(filename string, content []byte) *multipart.FileHeader { + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + part, _ := writer.CreateFormFile("file", filename) + part.Write(content) + writer.Close() + + reader := multipart.NewReader(body, writer.Boundary()) + form, _ := reader.ReadForm(int64(len(content)) + 1024) + + if files, ok := form.File["file"]; ok && len(files) > 0 { + return files[0] + } + + return nil +} + +func TestValidateFileExtension(t *testing.T) { + tests := []struct { + name string + filename string + contentType string + expectError bool + }{ + // Valid cases + {"Valid PNG", "image.png", "image/png", false}, + {"Valid JPEG", "photo.jpg", "image/jpeg", false}, + {"Valid JPEG alt", "photo.jpeg", "image/jpeg", false}, + {"Valid WebP", "image.webp", "image/webp", false}, + {"Valid SVG", "icon.svg", "image/svg+xml", false}, + {"Valid TIFF", "scan.tiff", "image/tiff", false}, + {"Valid TIFF alt", "scan.tif", "image/tiff", false}, + {"Valid MP4", "video.mp4", "video/mp4", false}, + {"Valid MPEG", "movie.mpeg", "video/mpeg", false}, + {"Valid OGG", "video.ogg", "video/ogg", false}, + {"Valid PDF", "document.pdf", "application/pdf", false}, + {"Valid TXT", "readme.txt", "text/plain", false}, + + // Case insensitive + {"Valid PNG uppercase", "IMAGE.PNG", "image/png", false}, + {"Valid JPG mixed case", "Photo.JpG", "image/jpeg", false}, + + // Invalid cases + {"Wrong extension for PNG", "image.jpg", "image/png", true}, + {"Wrong extension for PDF", "doc.txt", "application/pdf", true}, + {"Wrong extension for MP4", "video.avi", "video/mp4", true}, + {"Unsupported type", "file.exe", "application/octet-stream", true}, + {"No extension", "file", "image/png", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateFileExtension(tt.filename, tt.contentType) + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateFileSignature(t *testing.T) { + tests := []struct { + name string + filename string + contentType string + content []byte + expectError bool + }{ + // Valid signatures + { + name: "Valid PNG", + filename: "test.png", + contentType: "image/png", + content: []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D}, + expectError: false, + }, + { + name: "Valid JPEG JFIF", + filename: "test.jpg", + contentType: "image/jpeg", + content: []byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46}, + expectError: false, + }, + { + name: "Valid JPEG Exif", + filename: "photo.jpg", + contentType: "image/jpeg", + content: []byte{0xFF, 0xD8, 0xFF, 0xE1, 0x00, 0x00, 0x45, 0x78, 0x69, 0x66}, + expectError: false, + }, + { + name: "Valid WebP", + filename: "image.webp", + contentType: "image/webp", + content: []byte{0x52, 0x49, 0x46, 0x46, 0x00, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50}, + expectError: false, + }, + { + name: "Valid TIFF little-endian", + filename: "scan.tiff", + contentType: "image/tiff", + content: []byte{0x49, 0x49, 0x2A, 0x00, 0x08, 0x00, 0x00, 0x00}, + expectError: false, + }, + { + name: "Valid TIFF big-endian", + filename: "scan.tif", + contentType: "image/tiff", + content: []byte{0x4D, 0x4D, 0x00, 0x2A, 0x00, 0x00, 0x00, 0x08}, + expectError: false, + }, + { + name: "Valid SVG with XML declaration", + filename: "icon.svg", + contentType: "image/svg+xml", + content: []byte(""), + expectError: false, + }, + { + name: "Valid SVG with svg tag", + filename: "graphic.svg", + contentType: "image/svg+xml", + content: []byte(""), + expectError: false, + }, + { + name: "Valid PDF", + filename: "doc.pdf", + contentType: "application/pdf", + content: []byte("%PDF-1.4\n%����\n"), + expectError: false, + }, + { + name: "Valid plain text", + filename: "readme.txt", + contentType: "text/plain", + content: []byte("This is a plain text file"), + expectError: false, + }, + { + name: "Valid MPEG Program Stream", + filename: "video.mpeg", + contentType: "video/mpeg", + content: []byte{0x00, 0x00, 0x01, 0xBA, 0x44, 0x00, 0x04, 0x00}, + expectError: false, + }, + { + name: "Valid OGG", + filename: "video.ogg", + contentType: "video/ogg", + content: []byte{0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00}, + expectError: false, + }, + + // Invalid signatures + { + name: "Invalid PNG signature", + filename: "fake.png", + contentType: "image/png", + content: []byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46}, // JPEG bytes + expectError: true, + }, + { + name: "Invalid JPEG signature", + filename: "fake.jpg", + contentType: "image/jpeg", + content: []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, // PNG bytes + expectError: true, + }, + { + name: "Invalid WebP - missing WEBP marker", + filename: "fake.webp", + contentType: "image/webp", + content: []byte{0x52, 0x49, 0x46, 0x46, 0x00, 0x00, 0x00, 0x00, 0x46, 0x41, 0x4B, 0x45}, + expectError: true, + }, + { + name: "Invalid PDF signature", + filename: "fake.pdf", + contentType: "application/pdf", + content: []byte("This is not a PDF file"), + expectError: true, + }, + { + name: "File too small", + filename: "tiny.png", + contentType: "image/png", + content: []byte{0x89}, + expectError: true, + }, + { + name: "Empty file", + filename: "empty.jpg", + contentType: "image/jpeg", + content: []byte{}, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fileHeader := createTestFileHeader(tt.filename, tt.content) + if fileHeader == nil { + t.Fatal("Failed to create test file header") + } + + err := ValidateFileSignature(fileHeader, tt.contentType) + if tt.expectError { + assert.Error(t, err, "Expected error for %s", tt.name) + } else { + assert.NoError(t, err, "Expected no error for %s", tt.name) + } + }) + } +} + +func TestValidateFile(t *testing.T) { + tests := []struct { + name string + filename string + contentType string + content []byte + expectError bool + }{ + { + name: "Valid PNG file", + filename: "image.png", + contentType: "image/png", + content: []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D}, + expectError: false, + }, + { + name: "Valid JPEG file", + filename: "photo.jpg", + contentType: "image/jpeg", + content: []byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46}, + expectError: false, + }, + { + name: "Wrong extension", + filename: "image.jpg", + contentType: "image/png", + content: []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, + expectError: true, + }, + { + name: "Wrong signature", + filename: "image.png", + contentType: "image/png", + content: []byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46}, + expectError: true, + }, + { + name: "Valid PDF", + filename: "document.pdf", + contentType: "application/pdf", + content: []byte("%PDF-1.5\n"), + expectError: false, + }, + { + name: "Unsupported type", + filename: "app.exe", + contentType: "application/octet-stream", + content: []byte{0x4D, 0x5A}, // EXE signature + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fileHeader := createTestFileHeader(tt.filename, tt.content) + if fileHeader == nil { + t.Fatal("Failed to create test file header") + } + + err := ValidateFile(fileHeader, tt.contentType) + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateMP4Signature(t *testing.T) { + // MP4 has signature at offset 4 + validMP4 := []byte{ + 0x00, 0x00, 0x00, 0x20, // Box size + 0x66, 0x74, 0x79, 0x70, // 'ftyp' + 0x69, 0x73, 0x6F, 0x6D, // 'isom' + } + + fileHeader := createTestFileHeader("video.mp4", validMP4) + if fileHeader == nil { + t.Fatal("Failed to create test file header") + } + + err := ValidateFileSignature(fileHeader, "video/mp4") + assert.NoError(t, err, "Valid MP4 should pass validation") + + // Test invalid MP4 + invalidMP4 := []byte{0x00, 0x00, 0x00, 0x20, 0x46, 0x41, 0x4B, 0x45} + fileHeader = createTestFileHeader("fake.mp4", invalidMP4) + if fileHeader == nil { + t.Fatal("Failed to create test file header") + } + + err = ValidateFileSignature(fileHeader, "video/mp4") + assert.Error(t, err, "Invalid MP4 should fail validation") +}