From 0cb4ebc7997c209727254a56f79d1b99ce30dfbf Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Tue, 11 Aug 2026 19:07:55 +0200 Subject: [PATCH 01/14] feat(azure): add native Azure Blob Storage support and distributed leases --- .gitignore | 2 + CHANGELOG.md | 3 +- README.md | 3 + docs/AZURE_BLOB_STORAGE.md | 268 +++++ docs/S3_STORAGE.md | 46 +- pom.xml | 10 + .../tus/server/TusFileUploadService.java | 18 +- .../server/upload/UploadLockingService.java | 10 + .../server/upload/UploadStorageService.java | 9 + .../azure/AzureBlobConcatenationService.java | 220 +++++ .../upload/azure/AzureBlobLockingService.java | 308 ++++++ .../upload/azure/AzureBlobStorageService.java | 918 ++++++++++++++++++ .../upload/azure/AzureBlobUploadLock.java | 115 +++ .../server/upload/azure/AzureErrorType.java | 33 + .../tus/server/upload/azure/AzureUtils.java | 67 ++ ...adLocalCachedStorageAndLockingService.java | 7 + .../upload/disk/DiskLockingService.java | 47 +- .../upload/s3/S3ConcatenationService.java | 13 +- .../tus/server/upload/s3/S3ErrorType.java | 3 + .../server/upload/s3/S3LockingService.java | 48 +- .../server/upload/s3/S3StorageService.java | 21 +- .../desair/tus/server/upload/s3/S3Utils.java | 10 +- .../java/me/desair/tus/server/TestUtils.java | 43 + .../tus/server/TusFileUploadServiceTest.java | 11 + .../AzureBlobConcatenationServiceTest.java | 135 +++ .../azure/AzureBlobLockingServiceTest.java | 98 ++ .../azure/AzureBlobStorageServiceTest.java | 179 ++++ .../upload/azure/AzureBlobUploadLockTest.java | 62 ++ .../server/upload/azure/AzureUtilsTest.java | 79 ++ .../upload/azure/ITAzureBlobRufhProtocol.java | 59 ++ .../azure/ITAzureBlobStorageServiceTest.java | 128 +++ .../ITAzureBlobTusFileUploadService.java | 68 ++ ...calCachedStorageAndLockingServiceTest.java | 3 + .../upload/disk/DiskLockingServiceTest.java | 8 + .../upload/s3/S3ConcatenationServiceTest.java | 10 + .../upload/s3/S3LockingServiceTest.java | 5 + .../upload/s3/S3StorageServiceTest.java | 11 + .../tus/server/upload/s3/S3UtilsTest.java | 6 + 38 files changed, 3064 insertions(+), 20 deletions(-) create mode 100644 docs/AZURE_BLOB_STORAGE.md create mode 100644 src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java create mode 100644 src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java create mode 100644 src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java create mode 100644 src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java create mode 100644 src/main/java/me/desair/tus/server/upload/azure/AzureErrorType.java create mode 100644 src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java create mode 100644 src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java create mode 100644 src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java create mode 100644 src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java create mode 100644 src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java create mode 100644 src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java create mode 100644 src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobRufhProtocol.java create mode 100644 src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageServiceTest.java create mode 100644 src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobTusFileUploadService.java diff --git a/.gitignore b/.gitignore index 5fc7380f..814c5352 100644 --- a/.gitignore +++ b/.gitignore @@ -174,3 +174,5 @@ __pycache__/ *.pyc CONFORMITY_TEST_IMPROVEMENTS.md S3_STORAGE_ANALYSIS.md +AZURE_BLOB_STORAGE_ANALYSIS.md +AZURE_BLOB_STORAGE_IMPROVEMENTS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c6a25b31..873a7015 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,8 @@ All notable changes to this project will be documented in this file. ## [2.0.0] ### Added -- **S3-Compatible Storage & Distributed Locking**: Added native S3 storage support via `S3StorageService` (AWS SDK v2), distributed locking via `S3LockingService` (S3 conditional writes with TTL leases and interrupt signals for multi-replica container deployments), S3-native concatenation via `S3ConcatenationService`, and complete documentation in `docs/S3_STORAGE.md`. +- **S3-Compatible Storage & Distributed Locking**: Added native S3 storage support via `S3StorageService` (MinIO SDK), distributed locking via `S3LockingService` (S3 conditional writes with TTL leases and interrupt signals for multi-replica container deployments), S3-native concatenation via `S3ConcatenationService`, and complete documentation in `docs/S3_STORAGE.md`. +- **Azure Blob Storage & Distributed Leases**: Added native Azure Blob Storage support via `AzureBlobStorageService` (Block Blob staging with streaming appends, sub-threshold buffering, truncation, and deduplication), distributed locking via `AzureBlobLockingService` (Azure Blob Leases with auto-renewal, JVM interruption, cross-replica `.stop` signals, and clean shutdown), zero-copy server-side concatenation via `AzureBlobConcatenationService` (`stageBlockFromUrl`), and comprehensive documentation in `docs/AZURE_BLOB_STORAGE.md`. - **IETF Resumable Uploads for HTTP (RUFH) Protocol**: Implemented full support for the official IETF Resumable Uploads for HTTP specification (`draft-ietf-httpbis-resumable-upload-12`). - **Dual Protocol Auto-Detection**: Added transparent protocol routing in `TusFileUploadService` supporting both legacy `TUS_1_0_0` (`Tus-Resumable: 1.0.0`) and `RUFH` (`ProtocolVersion.RUFH`) clients concurrently on the same endpoint. - **RFC 9651 Structured Header Fields**: Implemented RFC 9651 parsing and serialization for `Upload-Offset`, `Upload-Complete`, `Upload-Length`, and `Upload-Limit` dictionary headers. diff --git a/README.md b/README.md index 6d4f13d5..ff3519c7 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,9 @@ The Javadoc of this library can be found at https://tus.desair.me/. As of versio 2. **S3-Compatible Object Storage** (`S3StorageService`, `S3LockingService`, & `S3ConcatenationService`): - **Cloud & On-Premise S3**: AWS S3, MinIO, Cloudflare R2, Ceph, or Google Cloud Storage. - **Multi-Replica Support**: Uses distributed S3 object locking and TTL leases, enabling multi-replica container deployments without requiring Redis or external databases. +3. **Azure Blob Storage** (`AzureBlobStorageService`, `AzureBlobLockingService`, & `AzureBlobConcatenationService`): + - **Microsoft Azure Cloud**: Native Azure Blob Storage using the `azure-storage-blob` SDK. + - **Multi-Replica Support**: Uses native Azure Blob Leases (30s renewable leases) for distributed locking across cluster replicas. See [Azure Blob Storage Guide](docs/AZURE_BLOB_STORAGE.md). ## Quick Start and Examples The tus-java-server library only depends on Jakarta Servlet API 6.0 and some Apache Commons utility libraries. This diff --git a/docs/AZURE_BLOB_STORAGE.md b/docs/AZURE_BLOB_STORAGE.md new file mode 100644 index 00000000..77aada19 --- /dev/null +++ b/docs/AZURE_BLOB_STORAGE.md @@ -0,0 +1,268 @@ +# Azure Blob Storage Support for `tus-java-server` + +`tus-java-server` provides native support for storing resumable file uploads in **Azure Blob Storage** using the official Microsoft Azure Storage Blob SDK (`com.azure:azure-storage-blob`). + +The implementation consists of four primary components: +- **`AzureBlobStorageService`** (implements `UploadStorageService`) — handles Block Blob uploads via staged block staging (`stageBlock` / `commitBlockList`), streaming appends, sub-threshold `.part` buffering, block list truncation, expiration, and checksum deduplication. +- **`AzureBlobLockingService`** (implements `UploadLockingService`) — provides distributed locking using native Azure Blob Leases (30s duration) on `.lock` target blobs with background renewal, enabling multi-replica container deployments without requiring Redis or external databases. +- **`AzureBlobUploadLock`** (implements `UploadLock`) — encapsulates active Azure Blob Leases with a background daemon thread that periodically renews the lease every 10 seconds. +- **`AzureBlobConcatenationService`** (implements `UploadConcatenationService`) — provides server-side zero-copy concatenation using Azure's native `stageBlockFromUrl` operation. + +--- + +## 1. Quick Start + +### Step 1: Add Dependencies + +Add the official Azure Storage Blob SDK and Jackson dependencies to your application's `pom.xml`: + +```xml + + + + com.azure + azure-storage-blob + 12.35.0 + + + + + com.fasterxml.jackson.core + jackson-databind + 2.22.1 + + +``` + +### Step 2: Configure `TusFileUploadService` + +```java +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobContainerClientBuilder; +import me.desair.tus.server.TusFileUploadService; +import me.desair.tus.server.upload.azure.AzureBlobStorageService; +import me.desair.tus.server.upload.azure.AzureBlobLockingService; + +// 1. Option A (Recommended Production Setup): Managed Identity via DefaultAzureCredential +String endpoint = System.getenv("AZURE_STORAGE_BLOB_ENDPOINT"); // e.g. "https://myaccount.blob.core.windows.net" +String containerName = System.getenv().getOrDefault("AZURE_STORAGE_CONTAINER", "uploads"); + +BlobContainerClient containerClient = new BlobContainerClientBuilder() + .endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .containerName(containerName) + .buildClient(); + +// Option B (Alternative Production Setup): Connection String from Secret Manager / Env Var +// String connectionString = System.getenv("AZURE_STORAGE_CONNECTION_STRING"); +// BlobContainerClient containerClient = new BlobContainerClientBuilder() +// .connectionString(connectionString) +// .containerName(containerName) +// .buildClient(); + +// 2. Instantiate Azure Blob Storage & Distributed Locking services +AzureBlobStorageService azureStorageService = new AzureBlobStorageService(containerClient); +AzureBlobLockingService azureLockingService = new AzureBlobLockingService(containerClient); + +// 3. Configure TusFileUploadService with Azure storage and locking +// Note: Automatic JVM shutdown hooks are built-in by default to terminate watchdog threads on pod exit. +// Manual call to tusService.close() or azureLockingService.close() is optional for custom container lifecycles. +TusFileUploadService tusService = new TusFileUploadService() + .withUploadUri("/files/upload") + .withUploadStorageService(azureStorageService) + .withUploadLockingService(azureLockingService); +``` + +--- + +## 2. Recommendation: Request Caching with `ThreadLocalCachedStorageAndLockingService` + +> [!IMPORTANT] +> **Why `ThreadLocalCachedStorageAndLockingService` is Recommended for Azure**: +> By default, `TusFileUploadService` automatically wraps your custom `UploadStorageService` and `UploadLockingService` in a `ThreadLocalCachedStorageAndLockingService`. +> +> During a single HTTP request lifecycle (POST, PATCH, HEAD, DELETE), the tus server validates request headers, reads upload state, appends data, and constructs response headers. Without caching, retrieving `UploadInfo` and calculating offsets would require multiple redundant network roundtrips to Azure (`downloadContent` on `.info`, `getProperties`). +> +> `ThreadLocalCachedStorageAndLockingService` caches the `UploadInfo` in thread-local memory for the duration of a single HTTP request, releasing the cache automatically when the upload lock is closed at the end of the request. This dramatically reduces Azure network latency and API call cost per request. + +--- + +## 3. Object Storage Layout + +`AzureBlobStorageService` uses a clean, structured blob naming convention: + +``` +/ +├── uploads/ # Final upload data (Block Blob) +├── metadata/.info # JSON-serialized UploadInfo +├── metadata/.part # Incomplete sub-threshold buffer blob +├── checksums// # Deduplication checksum index object +├── locks/.lock # Distributed lock target blob (Blob Lease) +└── locks/.stop # Cross-replica contention interrupt signal +``` + +### Key Prefix Defaults + +| Setting | Default Value | Description | +|---------|---------------|-------------| +| `uploadPrefix` | `"uploads/"` | Blob name prefix for final completed file objects | +| `metadataPrefix` | `"metadata/"` | Blob name prefix for `.info` JSON and `.part` buffers | +| `checksumsPrefix` | `"checksums/"` | Blob name prefix for deduplication index objects | +| `locksPrefix` | `"locks/"` | Blob name prefix for distributed lock lease objects | + +--- + +## 4. Post-Upload Processing (`getAzureBlobName`) + +After an upload completes, downstream services can obtain the direct Azure blob name of the final object using `getAzureBlobName(uploadUri, ownerKey)`: + +```java +import com.azure.storage.blob.BlobClient; +import me.desair.tus.server.upload.azure.AzureBlobStorageService; + +AzureBlobStorageService azureStorage = (AzureBlobStorageService) tusService.getUploadStorageService(); + +String uploadUri = "/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"; +String ownerKey = "user-123"; + +// 1. Obtain full Azure blob name after upload completion +String blobName = azureStorage.getAzureBlobName(uploadUri, ownerKey); +// e.g. "uploads/24249a5b-01a4-4bf8-b67a-364273bb5a2e" + +// 2. Direct Azure SDK access for post-upload processing +BlobClient dataBlob = containerClient.getBlobClient(blobName); +``` + +--- + +## 5. Custom Endpoints & Authentication Best Practices + +Since `AzureBlobStorageService` accepts a pre-configured `BlobContainerClient`, authentication is fully delegated to the user. + +### Production: `DefaultAzureCredential` (Managed Identity / Azure AD) + +```java +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobContainerClientBuilder; + +BlobContainerClient containerClient = new BlobContainerClientBuilder() + .endpoint("https://.blob.core.windows.net") + .containerName("tus-uploads") + .credential(new DefaultAzureCredentialBuilder().build()) + .buildClient(); +``` + +### Local Development: Connection String / Azurite Emulator + +```java +BlobContainerClient containerClient = new BlobContainerClientBuilder() + .connectionString("UseDevelopmentStorage=true") + .containerName("tus-uploads") + .buildClient(); +``` + +--- + +## 6. Local Disk Buffer & Block Size Auto-Calibration + +`AzureBlobStorageService` streams incoming PATCH payloads in chunks of `optimalBlockSize` into temporary files, staging each block to Azure as it completes. Peak disk usage per upload is capped at `1 × optimalBlockSize` (e.g. 8 MB). + +Block sizes auto-calibrate based on total upload size: +- **Baseline Preferred Size**: 8 MB (configurable via constructor) +- **Minimum Block Size**: 4 MB +- **Maximum Block Size**: 4000 MiB (Azure limit) +- **Maximum Blocks per Blob**: 50,000 (Azure limit) + +--- + +## 7. Multi-Replica Container Deployments (Azure Blob Leases & Renewal) + +### Lease Renewal Rationale +`AzureBlobLockingService` uses native Azure Blob Leases (30-second duration) for distributed locking. Because large file uploads can stream over several minutes or hours, `AzureBlobUploadLock` runs a background daemon thread that renews the lease every 10 seconds. If an application server crashes unexpectedly, the lease auto-expires after 30 seconds without requiring manual lock cleanup sweeps. + +### Lock Contention Resolution +Lock contention resolution operates on two levels: +1. **JVM-local**: Active `InterruptibleInputStream` instances are registered in a concurrent map and interrupted directly if a concurrent lock request arrives in the same JVM. +2. **Cross-replica**: A `.stop` signal blob (`locks/.stop`) is written to Azure Storage. A background watchdog thread polls for `.stop` blobs and interrupts active streams on other cluster nodes. + +--- + +## 8. Troubleshooting Guide + +| Issue / Error | Root Cause | Solution | +|---|---|---| +| **HTTP 409 Conflict** | Another process or cluster pod currently holds an active lease on the lock blob. | Normal behavior during concurrent PATCH/DELETE requests. Retry after lock release. | +| **HTTP 404 BlobNotFound** | The upload metadata `.info` blob does not exist or was expired/deleted. | Verify upload ID validity or upload expiration timestamps (`uploadExpirationPeriod`). | +| **Azurite connection refused** | Azurite emulator is not running or listening on port 10000. | Launch Azurite via Docker (`docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite`). | +| **`MaxAppendSizeExceededException`** | Incoming PATCH payload chunk exceeded the configured `maxAppendSize`. | Adjust `withMaxAppendSize()` setting on `TusFileUploadService`. | + +--- + +## 9. Test Suite Structure + +| Test Suite Class | Type | Dependencies | Execution Time | Description | +|---|---|---|---|---| +| `AzureBlobStorageServiceTest` | Unit Test | Mockito (Offline) | < 1s | Fast unit tests for storage CRUD, chunk streaming, and deduplication. | +| `AzureBlobLockingServiceTest` | Unit Test | Mockito (Offline) | < 1s | Unit tests for lease acquisition, lock contention, and stream interruption. | +| `AzureBlobUploadLockTest` | Unit Test | Mockito (Offline) | < 1s | Unit tests for lease release and background renewal. | +| `AzureBlobConcatenationServiceTest` | Unit Test | Mockito (Offline) | < 1s | Unit tests for zero-copy concatenation merging. | +| `ITAzureBlobStorageServiceTest` | Integration | Azurite (Docker) | ~ 5s | Live end-to-end storage integration test against Azurite emulator. | +| `ITAzureBlobRufhProtocol` | Integration | Azurite (Docker) | ~ 8s | IETF RUFH protocol integration suite for Azure backend. | +| `ITAzureBlobTusFileUploadService` | Integration | Azurite (Docker) | ~ 8s | Tus 1.0.0 protocol integration suite for Azure backend. | + +--- + +## 10. Security & RBAC Permissions Policy + +1. **Authentication**: Use `DefaultAzureCredential` or Managed Identity in production. Never hardcode storage account keys in source code. +2. **RBAC Data-Plane Role**: Assign the **`Storage Blob Data Contributor`** role to the application identity. +3. **Lease Permissions Note**: Note that native Azure Blob Lease operations (`acquireLease`, `renewLease`, `releaseLease`) require the `Microsoft.Storage/storageAccounts/blobServices/containers/blobs/write` data action in Azure RBAC policies. + +### Minimal RBAC Policy (JSON) + +```json +{ + "properties": { + "roleName": "TusFileUploadServiceBlobDataContributor", + "description": "Minimum RBAC permissions for tus-java-server Azure Blob Storage integration", + "assignableScopes": [ + "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts/" + ], + "permissions": [ + { + "actions": [], + "notActions": [], + "dataActions": [ + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/read", + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/write", + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/delete", + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/add/action" + ], + "notDataActions": [] + } + ] + } +} +``` + +--- + +## 11. Operational Hardening & Cost Optimization Guidance + +1. **Storage Lifecycle Management Policies**: Configure an Azure Lifecycle Management policy to automatically delete uncommitted block blobs or orphaned `.part` buffers older than 7 days. +2. **Container Soft Delete & Versioning**: Enable Azure Container Soft Delete (e.g. 7-day retention) to protect completed upload data from accidental deletion. +3. **API Cost Optimization**: `AzureBlobStorageService` minimizes API costs by combining GET calls, using single `getProperties()` lookups, and caching metadata in `ThreadLocalCachedStorageAndLockingService`. + +--- + +## 12. Running Local Azure Integration Tests (Azurite) + +```bash +# Run fast offline unit tests +mvn test -Dtest="*Azure*Test" -q + +# Run integration tests against Azurite container +mvn test -Dtest="ITAzureBlob*" -q +``` diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index 5dd34a69..80b3503a 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -46,17 +46,28 @@ import me.desair.tus.server.TusFileUploadService; import me.desair.tus.server.upload.s3.S3StorageService; import me.desair.tus.server.upload.s3.S3LockingService; -// 1. Instantiate MinIO Client for AWS S3 or S3-compatible storage +// 1. Production Configuration: Load S3 parameters securely from environment variables +String endpoint = System.getenv().getOrDefault("S3_ENDPOINT", "https://s3.us-east-1.amazonaws.com"); +String bucketName = System.getenv().getOrDefault("S3_BUCKET_NAME", "my-upload-bucket"); +String accessKey = System.getenv("AWS_ACCESS_KEY_ID"); +String secretKey = System.getenv("AWS_SECRET_ACCESS_KEY"); + MinioClient minioClient = MinioClient.builder() - .endpoint("https://s3.amazonaws.com") - .credentials("YOUR_ACCESS_KEY", "YOUR_SECRET_KEY") + .endpoint(endpoint) + .credentials(accessKey, secretKey) .build(); -// 2. Configure TusFileUploadService with S3 storage and locking +// 2. Instantiate S3 Storage and Distributed Locking services +S3StorageService s3StorageService = new S3StorageService(minioClient, bucketName); +S3LockingService s3LockingService = new S3LockingService(minioClient, bucketName); + +// 3. Configure TusFileUploadService with S3 storage and locking +// Note: Automatic JVM shutdown hooks are built-in by default to terminate watchdog threads on pod exit. +// Manual call to tusService.close() or s3LockingService.close() is optional for custom container lifecycles. TusFileUploadService tusService = new TusFileUploadService() .withUploadUri("/files/upload") - .withUploadStorageService(new S3StorageService(minioClient, "my-upload-bucket")) - .withUploadLockingService(new S3LockingService(minioClient, "my-upload-bucket")); + .withUploadStorageService(s3StorageService) + .withUploadLockingService(s3LockingService); ``` --- @@ -294,3 +305,26 @@ When the test suite executes: - **Port Conflicts**: Testcontainers dynamically binds MinIO to random available host ports, preventing port collision with existing local services. --- + +## 10. Troubleshooting Guide + +| Issue / Error | Root Cause | Solution | +|---------------|------------|----------| +| `UploadAlreadyLockedException` | Concurrent request to an active upload ID | Wait for current request to finish or ensure single-client sequencing | +| `NoSuchKey` / 404 on `.info` | Expiration or upload terminated | Client must re-initiate upload creation via `POST` | +| High S3 Request Costs | Uncached `UploadInfo` lookups | Ensure `ThreadLocalCachedStorageAndLockingService` wrapper is enabled | +| Thread leak on app shutdown | `S3LockingService` watchdog executor active | Call `s3LockingService.close()` on application shutdown | + +--- + +## 11. Operational Hardening & S3 Cost Guidelines + +### S3 Bucket Lifecycle Rules +To automatically clean up abandoned partial uploads or lock files in case of sudden server crashes, configure S3 Lifecycle Rules on your bucket: + +- **Expire Incomplete Multipart Uploads**: Set rule to abort incomplete multipart uploads after 1–7 days. +- **Expire `metadata/*.part` Objects**: Configure lifecycle expiration for objects under `metadata/` prefix matching `*.part` older than 7 days. +- **Expire `locks/*` Objects**: Configure lifecycle expiration for objects under `locks/` prefix older than 1 day. + +### IAM Data Action Summary +Ensure your IAM role or service account is granted `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject`, and `s3:ListBucket` permissions on your target bucket path. diff --git a/pom.xml b/pom.xml index be6b92fc..26936d61 100644 --- a/pom.xml +++ b/pom.xml @@ -59,6 +59,16 @@ 9.0.3 provided + + + + com.azure + azure-storage-blob + 12.35.0 + provided + + + com.squareup.okhttp3 okhttp diff --git a/src/main/java/me/desair/tus/server/TusFileUploadService.java b/src/main/java/me/desair/tus/server/TusFileUploadService.java index c11b8eae..23c77389 100644 --- a/src/main/java/me/desair/tus/server/TusFileUploadService.java +++ b/src/main/java/me/desair/tus/server/TusFileUploadService.java @@ -2,6 +2,7 @@ import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -42,7 +43,7 @@ import org.slf4j.LoggerFactory; /** Helper class that implements the server side tus v1.0.0 upload protocol */ -public class TusFileUploadService { +public class TusFileUploadService implements Closeable { public static final String TUS_API_VERSION = "1.0.0"; @@ -734,4 +735,19 @@ private void prepareCacheIfEnabled() { this.uploadLockingService = service; } } + + /** + * Closes underlying storage and locking services, releasing background threads and resources. + * + * @throws IOException If closing fails + */ + @Override + public void close() throws IOException { + if (uploadLockingService != null) { + uploadLockingService.close(); + } + if (uploadStorageService != null) { + uploadStorageService.close(); + } + } } diff --git a/src/main/java/me/desair/tus/server/upload/UploadLockingService.java b/src/main/java/me/desair/tus/server/upload/UploadLockingService.java index 4ddebc1d..78ab2613 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadLockingService.java @@ -61,4 +61,14 @@ default void registerInputStream(String requestUri, java.io.InputStream inputStr default void requestLockRelease(String requestUri) { // No-op by default for backwards compatibility } + + /** + * Closes resources and shuts down any background watchdog threads associated with this locking + * service. + * + * @throws IOException If closing fails + */ + default void close() throws IOException { + // No-op by default for backwards compatibility + } } diff --git a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java index 4c3672fc..65e55548 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java @@ -271,4 +271,13 @@ default void setJsonSerializationEnabled(boolean enabled) { default boolean isJsonSerializationEnabled() { return false; } + + /** + * Closes any underlying storage resources. + * + * @throws IOException If closing fails + */ + default void close() throws IOException { + // No-op by default for backward compatibility + } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java new file mode 100644 index 00000000..f858af58 --- /dev/null +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java @@ -0,0 +1,220 @@ +package me.desair.tus.server.upload.azure; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.specialized.BlockBlobClient; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import me.desair.tus.server.exception.UploadNotFoundException; +import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadStorageService; +import me.desair.tus.server.upload.concatenation.UploadConcatenationService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Server-side zero-copy {@link UploadConcatenationService} implementation for Azure Blob Storage. + * + *

Azure Concatenation Architecture: + * + *

    + *
  • Zero-Copy URL Block Staging ({@code stageBlockFromUrl}): Merges partial upload blobs + * into a final concatenated upload directly on the Azure Storage cluster using native + * block-copying by reference. Data is copied server-side on Azure without transferring + * payload bytes through application memory or network interfaces. + *
  • Fallback for Local Emulators: If {@code stageBlockFromUrl} returns 501 / + * APINotImplemented (e.g., when testing against local Azurite emulator), falls back to + * streamed block staging ({@code stageBlock}). + *
  • Atomic Commit: Commits the composed list of block IDs atomically using {@code + * commitBlockList}. + *
+ */ +public class AzureBlobConcatenationService implements UploadConcatenationService { + + private static final Logger log = LoggerFactory.getLogger(AzureBlobConcatenationService.class); + + private final BlobContainerClient containerClient; + private final String uploadPrefix; + private final UploadStorageService storageService; + + /** + * Constructs an {@link AzureBlobConcatenationService} with default upload prefix. + * + * @param containerClient Pre-configured Azure {@link BlobContainerClient} + * @param storageService Backing {@link UploadStorageService} instance + */ + public AzureBlobConcatenationService( + BlobContainerClient containerClient, UploadStorageService storageService) { + this(containerClient, AzureBlobStorageService.DEFAULT_OBJECT_PREFIX, storageService); + } + + /** + * Constructs an {@link AzureBlobConcatenationService} with custom upload prefix. + * + * @param containerClient Pre-configured Azure {@link BlobContainerClient} + * @param uploadPrefix Key prefix for data blobs + * @param storageService Backing {@link UploadStorageService} instance + */ + public AzureBlobConcatenationService( + BlobContainerClient containerClient, + String uploadPrefix, + UploadStorageService storageService) { + this.containerClient = + Objects.requireNonNull(containerClient, "containerClient must not be null"); + this.uploadPrefix = sanitizePrefix(uploadPrefix); + this.storageService = Objects.requireNonNull(storageService, "storageService must not be null"); + } + + @Override + public void merge(UploadInfo finalUpload) throws IOException, UploadNotFoundException { + if (finalUpload == null + || !finalUpload.isUploadInProgress() + || finalUpload.getConcatenationPartIds() == null) { + return; + } + + Long expirationPeriod = + storageService != null ? storageService.getUploadExpirationPeriod() : null; + List partialUploads = getPartialUploads(finalUpload); + + Long totalLength = calculateTotalLength(partialUploads); + boolean completed = checkAllCompleted(expirationPeriod, partialUploads); + + if (totalLength != null && totalLength > 0 && completed) { + List blockIds = new ArrayList<>(); + BlockBlobClient finalBlockBlob = + containerClient.getBlobClient(uploadPrefix + finalUpload.getId()).getBlockBlobClient(); + + int sequence = 0; + for (UploadInfo partialInfo : partialUploads) { + String blockId = generateBlockId(sequence++); + BlobClient partialBlob = containerClient.getBlobClient(uploadPrefix + partialInfo.getId()); + + try { + // 1. Attempt zero-copy server-side block copying on Azure Storage cluster + finalBlockBlob.stageBlockFromUrl(blockId, partialBlob.getBlobUrl(), null); + } catch (BlobStorageException e) { + // 2. Fallback to stream staging if stageBlockFromUrl is not implemented by emulator + AzureErrorType errorType = AzureUtils.parseErrorResponse(e); + if (errorType == AzureErrorType.API_NOT_IMPLEMENTED || e.getStatusCode() == 400) { + try (InputStream partIs = storageService.getUploadedBytes(partialInfo.getId())) { + finalBlockBlob.stageBlock(blockId, partIs, partialInfo.getOffset()); + } + } else { + throw e; + } + } + blockIds.add(blockId); + } + + // 3. Atomically commit block list on Azure Storage + finalBlockBlob.commitBlockList(blockIds, true); + + // Clean up any sub-threshold .part blob created during upload instantiation + try { + containerClient + .getBlobClient(uploadPrefix + finalUpload.getId() + ".part") + .deleteIfExists(); + } catch (Exception ignored) { + } + + // 4. Update final upload attributes + finalUpload.setOffset(totalLength); + finalUpload.setLength(totalLength); + finalUpload.setStorageUploadId(uploadPrefix + finalUpload.getId()); + if (expirationPeriod != null) { + finalUpload.updateExpiration(expirationPeriod); + } + storageService.update(finalUpload); + + log.info( + "Successfully merged {} partial uploads into concatenated upload {}", + partialUploads.size(), + finalUpload.getId()); + } + } + + @Override + public InputStream getConcatenatedBytes(UploadInfo info) + throws IOException, UploadNotFoundException { + if (info == null) { + throw new UploadNotFoundException("UploadInfo must not be null"); + } + + if (info.isUploadInProgress()) { + merge(info); + } + + if (!info.isUploadInProgress() && storageService != null) { + return storageService.getUploadedBytes(info.getId()); + } + + return new ByteArrayInputStream(new byte[0]); + } + + @Override + public List getPartialUploads(UploadInfo info) + throws IOException, UploadNotFoundException { + if (info == null || info.getConcatenationPartIds() == null) { + return Collections.emptyList(); + } + + List result = new ArrayList<>(); + for (String partUri : info.getConcatenationPartIds()) { + UploadInfo partInfo = storageService.getUploadInfo(partUri, info.getOwnerKey()); + if (partInfo == null) { + throw new UploadNotFoundException( + "Partial upload with URI " + partUri + " not found for concatenated upload"); + } + result.add(partInfo); + } + return result; + } + + private Long calculateTotalLength(List partialUploads) { + if (partialUploads == null || partialUploads.isEmpty()) { + return null; + } + long total = 0L; + for (UploadInfo info : partialUploads) { + if (info == null || info.getLength() == null) { + return null; + } + total += info.getLength(); + } + return total; + } + + private boolean checkAllCompleted(Long expirationPeriod, List partialUploads) { + if (partialUploads == null || partialUploads.isEmpty()) { + return false; + } + for (UploadInfo info : partialUploads) { + if (info == null || info.isUploadInProgress() || info.isExpired()) { + return false; + } + } + return true; + } + + private String generateBlockId(int index) { + String idString = String.format("concat-%06d", index); + return Base64.getEncoder().encodeToString(idString.getBytes(StandardCharsets.UTF_8)); + } + + private String sanitizePrefix(String prefix) { + if (prefix == null || prefix.isEmpty()) { + return ""; + } + String result = prefix.startsWith("/") ? prefix.substring(1) : prefix; + return result.endsWith("/") ? result : result + "/"; + } +} diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java new file mode 100644 index 00000000..f6df3ab2 --- /dev/null +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java @@ -0,0 +1,308 @@ +package me.desair.tus.server.upload.azure; + +import com.azure.core.util.BinaryData; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.models.BlobProperties; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.specialized.BlobLeaseClient; +import com.azure.storage.blob.specialized.BlobLeaseClientBuilder; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.lang.ref.WeakReference; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import me.desair.tus.server.exception.TusException; +import me.desair.tus.server.exception.UploadAlreadyLockedException; +import me.desair.tus.server.upload.UploadId; +import me.desair.tus.server.upload.UploadIdFactory; +import me.desair.tus.server.upload.UploadLock; +import me.desair.tus.server.upload.UploadLockingService; +import me.desair.tus.server.upload.UuidUploadIdFactory; +import me.desair.tus.server.util.InterruptibleInputStream; +import org.apache.commons.lang3.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Distributed {@link UploadLockingService} backed by Azure Blob Storage Leases. + * + *

Azure Distributed Locking & Contention Mechanics: + * + *

    + *
  • Azure Blob Leases: Locks are backed by 30-second native Azure Blob Leases on + * dedicated lock target blobs (e.g. {@code locks/.lock}). If another pod or thread + * attempts to acquire a lease on a locked blob, Azure returns HTTP 409 Conflict, which is + * translated to an {@link UploadAlreadyLockedException}. + *
  • Heartbeat Lease Renewal: Active locks automatically renew their lease via a + * background renewal thread in {@link AzureBlobUploadLock}, keeping the lock alive during + * long uploads. + *
  • Auto-Expiry on Node Crash: If a pod crashes unexpectedly (e.g., OOM or {@code kill + * -9}), Azure automatically releases the lease after 30 seconds, preventing permanent + * deadlocks. + *
  • Cross-Pod Contention & Interruption: When a concurrent request (e.g., HEAD or + * DELETE) arrives for a locked upload, the service interrupts local streams and writes a + * {@code locks/.stop} signal blob to Azure. A background watchdog thread detects + * the {@code .stop} file and interrupts streaming on remote pods. + *
+ */ +public class AzureBlobLockingService implements UploadLockingService, Closeable { + + private static final Logger log = LoggerFactory.getLogger(AzureBlobLockingService.class); + + public static final String DEFAULT_LOCKS_PREFIX = "locks/"; + private static final int LEASE_DURATION_SECONDS = 30; + + private final BlobContainerClient containerClient; + private final String locksPrefix; + private final Map> activeStreams = + new ConcurrentHashMap<>(); + + private UploadIdFactory idFactory = new UuidUploadIdFactory(); + + private Thread watchdogThread = null; + private final Object watchdogLock = new Object(); + + private final Thread shutdownHook; + private volatile boolean closed = false; + + /** + * Constructs an {@link AzureBlobLockingService} with default lock key prefix. + * + * @param containerClient Pre-configured Azure {@link BlobContainerClient} + */ + public AzureBlobLockingService(BlobContainerClient containerClient) { + this(containerClient, DEFAULT_LOCKS_PREFIX); + } + + /** + * Constructs an {@link AzureBlobLockingService} with customizable lock key prefix. + * + * @param containerClient Pre-configured Azure {@link BlobContainerClient} + * @param locksPrefix Blob name prefix for lock objects + */ + public AzureBlobLockingService(BlobContainerClient containerClient, String locksPrefix) { + this.containerClient = + Objects.requireNonNull(containerClient, "containerClient must not be null"); + this.locksPrefix = sanitizePrefix(locksPrefix); + + // Register automatic JVM shutdown hook to clean up watchdog threads on app/pod shutdown + this.shutdownHook = new Thread(this::closeQuietly, "azure-lock-shutdown-hook"); + registerShutdownHook(); + } + + private void registerShutdownHook() { + try { + Runtime.getRuntime().addShutdownHook(shutdownHook); + } catch (IllegalStateException ignored) { + // JVM is already shutting down + } + } + + private void deregisterShutdownHook() { + try { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } catch (IllegalStateException ignored) { + // JVM is already shutting down + } + } + + private void closeQuietly() { + try { + close(); + } catch (Exception ignored) { + } + } + + @Override + public void close() throws IOException { + if (!closed) { + closed = true; + deregisterShutdownHook(); + synchronized (watchdogLock) { + if (watchdogThread != null) { + watchdogThread.interrupt(); + watchdogThread = null; + } + } + activeStreams.clear(); + } + } + + @Override + public void setIdFactory(UploadIdFactory idFactory) { + this.idFactory = Objects.requireNonNull(idFactory, "idFactory must not be null"); + } + + @Override + public UploadLock lockUploadByUri(String requestUri) throws TusException, IOException { + UploadId uploadId = idFactory.readUploadId(requestUri); + if (uploadId == null) { + return null; + } + String idStr = uploadId.toString(); + + // 1. Ensure lock target blob exists on Azure Storage under locksPrefix + BlobClient lockBlob = containerClient.getBlobClient(locksPrefix + idStr + ".lock"); + ensureLockBlobExists(lockBlob); + + // 2. Instantiate Azure Blob Lease client for target lock blob + BlobLeaseClient leaseClient = new BlobLeaseClientBuilder().blobClient(lockBlob).buildClient(); + + try { + // 3. Acquire 30-second exclusive lease from Azure Blob Storage + leaseClient.acquireLease(LEASE_DURATION_SECONDS); + + // Lock successfully acquired: clear any lingering .stop signal blob + deleteStopSignalBlob(idStr); + + return new AzureBlobUploadLock(leaseClient, lockBlob, requestUri); + } catch (BlobStorageException e) { + AzureErrorType errorType = AzureUtils.parseErrorResponse(e); + if (errorType == AzureErrorType.LEASE_ALREADY_PRESENT + || errorType == AzureErrorType.CONFLICT) { + log.info("Lock contention for upload URI {}: Azure blob lease is already held", requestUri); + throw new UploadAlreadyLockedException( + "Upload with URI " + requestUri + " is currently locked"); + } + throw new IOException("Failed to acquire Azure blob lease lock for URI " + requestUri, e); + } + } + + @Override + public void cleanupStaleLocks() throws IOException { + // Azure Blob Leases auto-expire after 30s on holder failure; no manual sweeps needed + } + + @Override + public boolean isLocked(UploadId id) { + if (id == null) { + return false; + } + BlobClient lockBlob = containerClient.getBlobClient(locksPrefix + id + ".lock"); + try { + // Single HEAD call to fetch properties and check if LeaseState is "leased" + BlobProperties props = lockBlob.getProperties(); + return props.getLeaseState() != null + && Strings.CS.equals(props.getLeaseState().toString(), "leased"); + } catch (Exception e) { + return false; + } + } + + @Override + public void registerInputStream(String requestUri, InputStream inputStream) { + UploadId uploadId = idFactory.readUploadId(requestUri); + if (uploadId != null && inputStream instanceof InterruptibleInputStream) { + activeStreams.put( + uploadId.toString(), new WeakReference<>((InterruptibleInputStream) inputStream)); + ensureWatchdogRunning(); + } + } + + @Override + public void requestLockRelease(String requestUri) { + UploadId uploadId = idFactory.readUploadId(requestUri); + if (uploadId != null) { + String idStr = uploadId.toString(); + // 1. Interrupt active local input stream in JVM + interruptLocalStream(idStr); + // 2. Write cross-pod .stop signal blob to notify remote pods + createStopSignalBlob(idStr); + } + } + + /** Interrupts active JVM-local input stream for the given upload ID. */ + private void interruptLocalStream(String idStr) { + WeakReference streamRef = activeStreams.remove(idStr); + if (streamRef != null) { + InterruptibleInputStream stream = streamRef.get(); + if (stream != null) { + log.info("Interrupting JVM-local stream for upload ID {}", idStr); + stream.interrupt(); + } + } + } + + /** Creates a .stop signal blob to request remote pods to halt active streaming appends. */ + private void createStopSignalBlob(String idStr) { + try { + BlobClient stopBlob = containerClient.getBlobClient(locksPrefix + idStr + ".stop"); + stopBlob.upload(BinaryData.fromString("stop"), true); + } catch (Exception e) { + log.debug("Failed to write .stop signal blob for upload ID {}: {}", idStr, e.getMessage()); + } + } + + /** Deletes the .stop signal blob after lock acquisition. */ + private void deleteStopSignalBlob(String idStr) { + try { + BlobClient stopBlob = containerClient.getBlobClient(locksPrefix + idStr + ".stop"); + stopBlob.deleteIfExists(); + } catch (Exception ignored) { + // Ignore cleanup exceptions + } + } + + /** Ensures the lock target blob exists on Azure Blob Storage. */ + private void ensureLockBlobExists(BlobClient lockBlob) { + try { + if (!lockBlob.exists()) { + lockBlob.upload(BinaryData.fromBytes("lock".getBytes(StandardCharsets.UTF_8)), false); + } + } catch (BlobStorageException e) { + AzureErrorType errorType = AzureUtils.parseErrorResponse(e); + if (errorType != AzureErrorType.CONFLICT + && errorType != AzureErrorType.LEASE_ALREADY_PRESENT + && errorType != AzureErrorType.PRECONDITION_FAILED) { + log.debug("Lock target blob existence check: {}", e.getMessage()); + } + } catch (Exception e) { + log.debug("Lock target blob creation: {}", e.getMessage()); + } + } + + /** Ensures background watchdog thread is active for polling .stop signal blobs. */ + private void ensureWatchdogRunning() { + synchronized (watchdogLock) { + if (watchdogThread == null || !watchdogThread.isAlive()) { + watchdogThread = new Thread(this::pollStopSignals, "azure-lock-watchdog"); + watchdogThread.setDaemon(true); + watchdogThread.start(); + } + } + } + + /** Polls for .stop signal blobs every 2 seconds while active streams exist. */ + private void pollStopSignals() { + while (!Thread.currentThread().isInterrupted() && !activeStreams.isEmpty()) { + try { + Thread.sleep(2000L); + for (String idStr : activeStreams.keySet()) { + BlobClient stopBlob = containerClient.getBlobClient(locksPrefix + idStr + ".stop"); + if (stopBlob.exists()) { + log.info("Detected remote .stop signal blob for upload ID {}", idStr); + interruptLocalStream(idStr); + stopBlob.deleteIfExists(); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } catch (Exception e) { + log.debug("Error in azure-lock-watchdog polling loop: {}", e.getMessage()); + } + } + } + + private String sanitizePrefix(String prefix) { + if (prefix == null || prefix.isEmpty()) { + return ""; + } + String result = prefix.startsWith("/") ? prefix.substring(1) : prefix; + return result.endsWith("/") ? result : result + "/"; + } +} diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java new file mode 100644 index 00000000..45be0a12 --- /dev/null +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java @@ -0,0 +1,918 @@ +package me.desair.tus.server.upload.azure; + +import com.azure.core.util.BinaryData; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.Block; +import com.azure.storage.blob.models.BlockList; +import com.azure.storage.blob.models.BlockListType; +import com.azure.storage.blob.models.ListBlobsOptions; +import com.azure.storage.blob.specialized.BlockBlobClient; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.SequenceInputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Objects; +import me.desair.tus.server.checksum.ChecksumAlgorithm; +import me.desair.tus.server.exception.MaxAppendSizeExceededException; +import me.desair.tus.server.exception.MinAppendSizeNotMetException; +import me.desair.tus.server.exception.TusException; +import me.desair.tus.server.exception.UploadNotFoundException; +import me.desair.tus.server.upload.UploadId; +import me.desair.tus.server.upload.UploadIdFactory; +import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLockingService; +import me.desair.tus.server.upload.UploadStorageService; +import me.desair.tus.server.upload.UuidUploadIdFactory; +import me.desair.tus.server.upload.concatenation.UploadConcatenationService; +import me.desair.tus.server.util.UploadInfoJsonSerializer; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.input.BoundedInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Azure Blob Storage implementation of {@link UploadStorageService}. + * + *

Azure Architecture Overview: + * + *

    + *
  • Block Blobs & Staged Blocks: Upload data is stored using Azure Block Blobs, which + * consist of up to 50,000 uncommitted staged blocks ({@code stageBlock}) that are committed + * atomically via {@code commitBlockList}. + *
  • Sub-Threshold Buffering ({@code .part}): Appends smaller than the optimal block size + * (8 MB default) are buffered in a temporary {@code .part} blob under {@code metadata/} until + * a full block accumulates or the upload finishes. + *
  • Metadata ({@code .info}): Upload metadata is stored as JSON-serialized {@link + * UploadInfo} objects under {@code metadata/.info}. + *
  • Checksum Deduplication Index: Completed uploads are indexed by checksum under {@code + * checksums//}. Duplicate uploads link to the parent upload ID. + *
+ */ +public class AzureBlobStorageService implements UploadStorageService { + + private static final Logger log = LoggerFactory.getLogger(AzureBlobStorageService.class); + + public static final String DEFAULT_OBJECT_PREFIX = "uploads/"; + public static final String DEFAULT_METADATA_PREFIX = "metadata/"; + public static final String DEFAULT_CHECKSUMS_PREFIX = "checksums/"; + public static final String DEFAULT_LOCKS_PREFIX = "locks/"; + + // Azure Block Blob Constants & Auto-Calibration Limits + private static final long MIN_BLOCK_SIZE = 4L * 1024 * 1024; // 4 MB (minimum recommended floor) + private static final long DEFAULT_PREFERRED_BLOCK_SIZE = 8L * 1024 * 1024; // 8 MB + private static final long MAX_BLOCK_SIZE = 4000L * 1024 * 1024; // 4000 MiB (Azure Blob limit) + private static final int MAX_BLOCKS_PER_BLOB = 50_000; // Azure Block Blob block count limit + + private final BlobContainerClient containerClient; + private final String uploadPrefix; + private final String metadataPrefix; + private final String checksumsPrefix; + private final String locksPrefix; + private final Path tempBufferDir; + + private long preferredBlockSize = DEFAULT_PREFERRED_BLOCK_SIZE; + + private Long maxUploadSize; + private Long maxAppendSize; + private Long minAppendSize; + private Long minSize; + private Long uploadExpirationPeriod; + private boolean deduplicationEnabled = false; + + private UploadIdFactory idFactory = new UuidUploadIdFactory(); + private UploadConcatenationService concatenationService; + + /** + * Constructs an {@link AzureBlobStorageService} using default object key prefixes and system temp + * buffer directory. + * + * @param containerClient Pre-configured Azure {@link BlobContainerClient} + */ + public AzureBlobStorageService(BlobContainerClient containerClient) { + this( + containerClient, + DEFAULT_OBJECT_PREFIX, + DEFAULT_METADATA_PREFIX, + DEFAULT_CHECKSUMS_PREFIX, + DEFAULT_LOCKS_PREFIX, + Paths.get(System.getProperty("java.io.tmpdir"), "tus-azure-buffer")); + } + + /** + * Constructs an {@link AzureBlobStorageService} with fully customizable prefixes and buffer path. + * + * @param containerClient Pre-configured Azure {@link BlobContainerClient} + * @param uploadPrefix Key prefix for final data objects + * @param metadataPrefix Key prefix for metadata (.info and .part) objects + * @param checksumsPrefix Key prefix for checksum deduplication index objects + * @param locksPrefix Key prefix for distributed lock objects + * @param tempBufferDir Local directory for staging chunk bytes before Azure upload + */ + public AzureBlobStorageService( + BlobContainerClient containerClient, + String uploadPrefix, + String metadataPrefix, + String checksumsPrefix, + String locksPrefix, + Path tempBufferDir) { + this.containerClient = + Objects.requireNonNull(containerClient, "containerClient must not be null"); + this.uploadPrefix = sanitizePrefix(uploadPrefix); + this.metadataPrefix = sanitizePrefix(metadataPrefix); + this.checksumsPrefix = sanitizePrefix(checksumsPrefix); + this.locksPrefix = sanitizePrefix(locksPrefix); + this.tempBufferDir = Objects.requireNonNull(tempBufferDir, "tempBufferDir must not be null"); + + ensureDirectoryExists(this.tempBufferDir); + + this.concatenationService = + new AzureBlobConcatenationService(containerClient, this.uploadPrefix, this); + } + + @Override + public void setIdFactory(UploadIdFactory idFactory) { + this.idFactory = Objects.requireNonNull(idFactory, "idFactory must not be null"); + } + + @Override + public String getUploadUri() { + return idFactory.getUploadUri(); + } + + @Override + public UploadInfo create(UploadInfo info, String ownerKey) throws IOException { + Objects.requireNonNull(info, "UploadInfo must not be null"); + + // 1. Generate new unique UploadId and initialize upload attributes + UploadId id = idFactory.createId(); + info.setId(id); + info.setOwnerKey(ownerKey); + info.setOffset(0L); + + // Set storageUploadId to the actual Azure Blob identifier (e.g. "uploads/") + info.setStorageUploadId(getAzureBlobName(info)); + + if (uploadExpirationPeriod != null && uploadExpirationPeriod > 0) { + info.setExpirationTimestamp(System.currentTimeMillis() + uploadExpirationPeriod); + } + + // 2. Persist JSON metadata object to Azure (.info blob) + saveUploadInfo(info); + + // 3. If initial creation specifies 0 bytes, commit empty Block Blob immediately + if (info.getLength() != null && info.getLength() == 0) { + commitEmptyDataBlob(id); + checkAndApplyDeduplication(info); + } + + log.debug("Created new upload with ID {} for owner {}", id, ownerKey); + return info; + } + + @Override + public UploadInfo append(UploadInfo upload, InputStream inputStream) + throws IOException, TusException { + Objects.requireNonNull(upload, "UploadInfo must not be null"); + Objects.requireNonNull(inputStream, "InputStream must not be null"); + + // 1. Locate the incomplete sub-threshold .part blob buffer and query its current size + BlobClient partBlob = containerClient.getBlobClient(metadataPrefix + upload.getId() + ".part"); + long existingPartSize = getPartBlobSize(partBlob); + + // 2. Auto-calibrate optimal block size (4 MB floor up to 4000 MiB limit based on upload length) + long optimalBlockSize = calcOptimalBlockSize(upload.getLength()); + Long effectiveMaxAppendSize = getMaxAppendSize(); + + // 3. Obtain Azure Block Blob client and fetch pre-existing committed block list + BlockBlobClient blockBlobClient = + containerClient.getBlobClient(getAzureBlobName(upload)).getBlockBlobClient(); + List blockIds = getCommittedBlockIds(blockBlobClient); + + long totalAppended = 0L; + boolean streamFinished = false; + + File firstChunkFile = File.createTempFile("tus-azure-chunk-", ".tmp", tempBufferDir.toFile()); + try { + // 4. Read first chunk from incoming payload stream into local disk buffer + long firstChunkSize = readChunk(inputStream, firstChunkFile, optimalBlockSize); + totalAppended += firstChunkSize; + + validateMaxAppendSize(totalAppended, effectiveMaxAppendSize); + + long newOffset = upload.getOffset() + firstChunkSize; + boolean isUploadComplete = upload.getLength() != null && newOffset == upload.getLength(); + long totalBuffered = existingPartSize + firstChunkSize; + + if (firstChunkSize < optimalBlockSize && firstChunkSize >= 0) { + streamFinished = true; + } + + if (totalBuffered < optimalBlockSize && !isUploadComplete && streamFinished) { + // Small append under block size threshold: buffer data to .part blob directly + bufferToPartBlob(partBlob, existingPartSize, firstChunkFile, firstChunkSize); + } else { + // Data exceeds block size threshold: stage blocks to Azure Block Blob + stagePartBlobIfPresent(partBlob, existingPartSize, blockBlobClient, blockIds); + stageChunkFile(firstChunkFile, firstChunkSize, blockBlobClient, blockIds); + + // Process any remaining chunks from input stream + if (!streamFinished) { + totalAppended += + processRemainingChunks( + inputStream, + optimalBlockSize, + effectiveMaxAppendSize, + upload, + partBlob, + blockBlobClient, + blockIds, + totalAppended); + } + + // Commit updated block ID list on Azure so staged blocks become committed and readable + blockBlobClient.commitBlockList(blockIds, true); + } + + validateMinAppendSize(totalAppended); + + // 5. Update UploadInfo offset, expiration timestamp, and optional deduplication state + upload.setOffset(upload.getOffset() + totalAppended); + if (uploadExpirationPeriod != null && uploadExpirationPeriod > 0) { + upload.setExpirationTimestamp(System.currentTimeMillis() + uploadExpirationPeriod); + } + + boolean finalComplete = + upload.getLength() != null && upload.getOffset().equals(upload.getLength()); + if (finalComplete) { + checkAndApplyDeduplication(upload); + } + + saveUploadInfo(upload); + return upload; + } finally { + deleteFileQuietly(firstChunkFile); + } + } + + @Override + public UploadInfo getUploadInfo(String requestUri, String ownerKey) throws IOException { + UploadId id = idFactory.readUploadId(requestUri); + if (id == null) { + return null; + } + + UploadInfo info = getUploadInfo(id); + if (info == null) { + return null; + } + + // Owner key validation for access isolation + if (info.getOwnerKey() != null && !Objects.equals(ownerKey, info.getOwnerKey())) { + return null; + } + + return info; + } + + @Override + public UploadInfo getUploadInfo(UploadId id) throws IOException { + if (id == null) { + return null; + } + + // Single GET call attempt: download .info blob directly, catching 404 BlobStorageException + BlobClient infoBlob = containerClient.getBlobClient(metadataPrefix + id + ".info"); + try { + byte[] bytes = infoBlob.downloadContent().toBytes(); + String json = new String(bytes, StandardCharsets.UTF_8); + return UploadInfoJsonSerializer.deserialize(json); + } catch (BlobStorageException e) { + if (AzureUtils.parseErrorResponse(e) == AzureErrorType.BLOB_NOT_FOUND) { + return null; + } + throw new IOException("Failed to download upload info for ID " + id, e); + } catch (Exception e) { + log.debug("Error deserializing upload info for ID {}: {}", id, e.getMessage()); + return null; + } + } + + public String getAzureBlobName(UploadInfo uploadInfo) { + if (uploadInfo == null) { + return null; + } + // Resolve duplicate child uploads dynamically to parent upload blob + if (uploadInfo.getDuplicatesUploadId() != null) { + return uploadPrefix + uploadInfo.getDuplicatesUploadId().toString(); + } + return uploadPrefix + uploadInfo.getId().toString(); + } + + public String getAzureBlobName(String requestUri, String ownerKey) throws IOException { + UploadInfo info = getUploadInfo(requestUri, ownerKey); + return info != null ? getAzureBlobName(info) : null; + } + + @Override + public InputStream getUploadedBytes(String requestUri, String ownerKey) + throws IOException, UploadNotFoundException { + UploadInfo info = getUploadInfo(requestUri, ownerKey); + if (info == null) { + throw new UploadNotFoundException("Upload not found for URI " + requestUri); + } + return getUploadedBytes(info.getId()); + } + + @Override + public InputStream getUploadedBytes(UploadId id) throws IOException { + UploadInfo info = getUploadInfo(id); + if (info == null) { + return null; + } + // Read operations dynamically resolve duplicates to parent + String targetBlobName = getAzureBlobName(info); + BlockBlobClient blockBlobClient = + containerClient.getBlobClient(targetBlobName).getBlockBlobClient(); + BlobClient partBlob = containerClient.getBlobClient(metadataPrefix + info.getId() + ".part"); + + InputStream committedStream = null; + try { + if (Boolean.TRUE.equals(blockBlobClient.exists())) { + committedStream = blockBlobClient.openInputStream(); + } + } catch (Exception ignored) { + // Data blob does not exist yet (upload in progress under sub-threshold part buffer) + } + + InputStream partStream = null; + try { + if (Boolean.TRUE.equals(partBlob.exists()) && partBlob.getProperties().getBlobSize() > 0) { + partStream = partBlob.openInputStream(); + } + } catch (Exception ignored) { + // No sub-threshold part buffer present + } + + if (committedStream != null && partStream != null) { + return new SequenceInputStream(committedStream, partStream); + } else if (committedStream != null) { + return committedStream; + } else if (partStream != null) { + return partStream; + } else { + return new ByteArrayInputStream(new byte[0]); + } + } + + @Override + public void copyUploadTo(UploadInfo uploadInfo, OutputStream outputStream) + throws UploadNotFoundException, IOException { + Objects.requireNonNull(uploadInfo, "UploadInfo must not be null"); + Objects.requireNonNull(outputStream, "OutputStream must not be null"); + + try (InputStream is = getUploadedBytes(uploadInfo.getId())) { + if (is == null) { + throw new UploadNotFoundException("Upload data not found for ID " + uploadInfo.getId()); + } + IOUtils.copyLarge(is, outputStream); + } + } + + @Override + public void terminateUpload(UploadInfo uploadInfo) throws UploadNotFoundException, IOException { + if (uploadInfo == null || uploadInfo.getId() == null) { + return; + } + UploadId id = uploadInfo.getId(); + + // 1. Delete committed data blob + containerClient.getBlobClient(uploadPrefix + id).deleteIfExists(); + + // 2. Delete incomplete sub-threshold .part blob + containerClient.getBlobClient(metadataPrefix + id + ".part").deleteIfExists(); + + // 3. Delete metadata .info blob + containerClient.getBlobClient(metadataPrefix + id + ".info").deleteIfExists(); + + // 4. Delete checksum deduplication index blob if present + if (uploadInfo.getChecksum() != null && uploadInfo.getChecksumAlgorithm() != null) { + String checksumKey = + buildChecksumKey(uploadInfo.getChecksum(), uploadInfo.getChecksumAlgorithm()); + containerClient.getBlobClient(checksumKey).deleteIfExists(); + } + + // 5. Delete lock target and stop signal blobs (handling active lease exceptions gracefully) + try { + containerClient.getBlobClient(locksPrefix + id + ".stop").deleteIfExists(); + } catch (Exception ignored) { + } + + try { + containerClient.getBlobClient(locksPrefix + id + ".lock").deleteIfExists(); + } catch (Exception ignored) { + // Lock blob may be actively leased by current request lock (Azure 412 LeaseIdMissing) + } + + log.debug("Terminated upload with ID {}", id); + } + + @Override + public void removeLastNumberOfBytes(UploadInfo uploadInfo, long byteCount) + throws UploadNotFoundException, IOException { + Objects.requireNonNull(uploadInfo, "UploadInfo must not be null"); + if (byteCount <= 0) { + return; + } + + // Note: Per AGENTS.md §7, write/modify operations MUST NOT resolve duplicates to parent. + long currentOffset = uploadInfo.getOffset(); + long targetOffset = Math.max(0L, currentOffset - byteCount); + + BlockBlobClient blockBlobClient = + containerClient.getBlobClient(getAzureBlobName(uploadInfo)).getBlockBlobClient(); + BlobClient partBlob = + containerClient.getBlobClient(metadataPrefix + uploadInfo.getId() + ".part"); + + long blockBlobSize = 0L; + List committedBlocks = new ArrayList<>(); + try { + BlockList blockList = blockBlobClient.listBlocks(BlockListType.COMMITTED); + if (blockList != null && blockList.getCommittedBlocks() != null) { + committedBlocks = blockList.getCommittedBlocks(); + for (Block b : committedBlocks) { + blockBlobSize += b.getSizeLong(); + } + } + } catch (Exception ignored) { + // Blob doesn't exist or has no committed blocks + } + + if (targetOffset <= blockBlobSize) { + // Truncation cuts into committed blocks: delete .part blob completely + partBlob.deleteIfExists(); + + if (targetOffset == 0) { + blockBlobClient.deleteIfExists(); + } else { + File tempFile = + File.createTempFile("tus-azure-block-trim-", ".tmp", tempBufferDir.toFile()); + try { + try (InputStream is = + BoundedInputStream.builder() + .setInputStream(blockBlobClient.openInputStream()) + .setMaxCount(targetOffset) + .get(); + OutputStream os = new FileOutputStream(tempFile)) { + IOUtils.copyLarge(is, os); + } + String newBlockId = generateBlockId(0); + try (InputStream is = new java.io.BufferedInputStream(new FileInputStream(tempFile))) { + blockBlobClient.stageBlock(newBlockId, is, tempFile.length()); + } + blockBlobClient.commitBlockList(List.of(newBlockId), true); + } finally { + deleteFileQuietly(tempFile); + } + } + } else { + // Truncation only affects .part buffer: keep committed blocks, trim .part blob + long newPartSize = targetOffset - blockBlobSize; + if (newPartSize <= 0) { + partBlob.deleteIfExists(); + } else { + File tempFile = File.createTempFile("tus-azure-truncate-", ".tmp", tempBufferDir.toFile()); + try { + try (InputStream is = + BoundedInputStream.builder() + .setInputStream(partBlob.openInputStream()) + .setMaxCount(newPartSize) + .get(); + OutputStream os = new FileOutputStream(tempFile)) { + IOUtils.copyLarge(is, os); + } + partBlob.upload(BinaryData.fromFile(tempFile.toPath()), true); + } finally { + deleteFileQuietly(tempFile); + } + } + } + + uploadInfo.setOffset(targetOffset); + saveUploadInfo(uploadInfo); + } + + @Override + public UploadInfo getUploadInfoByChecksum(String checksum, ChecksumAlgorithm algorithm) + throws IOException { + if (!isUploadDeduplicationEnabled() || checksum == null || algorithm == null) { + return null; + } + + String checksumKey = buildChecksumKey(checksum, algorithm); + BlobClient checksumBlob = containerClient.getBlobClient(checksumKey); + + try { + byte[] bytes = checksumBlob.downloadContent().toBytes(); + String parentIdStr = new String(bytes, StandardCharsets.UTF_8).trim(); + UploadId parentId = new UploadId(parentIdStr); + UploadInfo parentInfo = getUploadInfo(parentId); + + // Self-cleaning index: if index points to missing/deleted upload, clean up stale index + if (parentInfo == null) { + checksumBlob.deleteIfExists(); + return null; + } + + return parentInfo; + } catch (BlobStorageException e) { + if (AzureUtils.parseErrorResponse(e) == AzureErrorType.BLOB_NOT_FOUND) { + return null; + } + throw new IOException("Error reading checksum index blob " + checksumKey, e); + } catch (Exception e) { + log.debug("Error retrieving upload info by checksum: {}", e.getMessage()); + return null; + } + } + + @Override + public void update(UploadInfo uploadInfo) throws IOException { + Objects.requireNonNull(uploadInfo, "UploadInfo must not be null"); + saveUploadInfo(uploadInfo); + } + + @Override + public void cleanupExpiredUploads(UploadLockingService lockingService) throws IOException { + cleanupExpiredUploads(); + } + + public void cleanupExpiredUploads() throws IOException { + ListBlobsOptions options = new ListBlobsOptions().setPrefix(metadataPrefix); + for (BlobItem item : containerClient.listBlobs(options, null)) { + if (item.getName().endsWith(".info")) { + String infoName = item.getName(); + String idStr = + infoName.substring(metadataPrefix.length(), infoName.length() - ".info".length()); + UploadId id = new UploadId(idStr); + UploadInfo info = getUploadInfo(id); + if (info != null && info.isExpired()) { + log.info("Cleaning up expired upload with ID {}", id); + try { + terminateUpload(info); + } catch (UploadNotFoundException ignored) { + } + } + } + } + } + + // --- Configuration Getters & Setters --- + + @Override + public void setMaxUploadSize(Long maxUploadSize) { + this.maxUploadSize = maxUploadSize; + } + + @Override + public long getMaxUploadSize() { + return maxUploadSize != null ? maxUploadSize : 0L; + } + + @Override + public void setMaxAppendSize(Long maxAppendSize) { + this.maxAppendSize = maxAppendSize; + } + + @Override + public Long getMaxAppendSize() { + return maxAppendSize != null ? maxAppendSize : (maxUploadSize != null ? maxUploadSize : null); + } + + @Override + public void setMinAppendSize(Long minAppendSize) { + this.minAppendSize = minAppendSize; + } + + @Override + public Long getMinAppendSize() { + return minAppendSize; + } + + @Override + public void setMinSize(Long minSize) { + this.minSize = minSize; + } + + @Override + public Long getMinSize() { + return minSize; + } + + @Override + public void setUploadExpirationPeriod(Long uploadExpirationPeriod) { + this.uploadExpirationPeriod = uploadExpirationPeriod; + } + + @Override + public Long getUploadExpirationPeriod() { + return uploadExpirationPeriod; + } + + @Override + public void setUploadDeduplicationEnabled(boolean deduplicationEnabled) { + this.deduplicationEnabled = deduplicationEnabled; + } + + @Override + public boolean isUploadDeduplicationEnabled() { + return deduplicationEnabled; + } + + @Override + public void setUploadConcatenationService(UploadConcatenationService concatenationService) { + this.concatenationService = concatenationService; + } + + @Override + public UploadConcatenationService getUploadConcatenationService() { + return concatenationService; + } + + public void setPreferredBlockSize(long preferredBlockSize) { + if (preferredBlockSize < MIN_BLOCK_SIZE || preferredBlockSize > MAX_BLOCK_SIZE) { + throw new IllegalArgumentException( + "preferredBlockSize must be between " + MIN_BLOCK_SIZE + " and " + MAX_BLOCK_SIZE); + } + this.preferredBlockSize = preferredBlockSize; + } + + public long getPreferredBlockSize() { + return preferredBlockSize; + } + + // --- Helper Methods --- + + /** Calculates auto-calibrated optimal block size based on total upload length. */ + private long calcOptimalBlockSize(Long totalLength) { + long size = preferredBlockSize; + if (totalLength != null && totalLength > 0 && totalLength / size >= MAX_BLOCKS_PER_BLOB) { + size = (totalLength / MAX_BLOCKS_PER_BLOB) + 1; + } + return Math.max(MIN_BLOCK_SIZE, Math.min(size, MAX_BLOCK_SIZE)); + } + + /** Saves UploadInfo object as JSON in .info metadata blob. */ + private void saveUploadInfo(UploadInfo info) throws IOException { + byte[] jsonBytes = UploadInfoJsonSerializer.serialize(info).getBytes(StandardCharsets.UTF_8); + BlobClient infoBlob = containerClient.getBlobClient(metadataPrefix + info.getId() + ".info"); + infoBlob.upload(BinaryData.fromBytes(jsonBytes), true); + } + + /** Gets size of existing .part blob using single HEAD call. */ + private long getPartBlobSize(BlobClient partBlob) { + try { + return partBlob.getProperties().getBlobSize(); + } catch (Exception e) { + return 0L; + } + } + + /** Reads up to maxBytes from InputStream into target File. */ + private long readChunk(InputStream is, File targetFile, long maxBytes) throws IOException { + long totalRead = 0L; + byte[] buffer = new byte[8192]; + try (FileOutputStream fos = new FileOutputStream(targetFile)) { + while (totalRead < maxBytes) { + int lenToRead = (int) Math.min(buffer.length, maxBytes - totalRead); + int read = is.read(buffer, 0, lenToRead); + if (read == -1) { + break; + } + fos.write(buffer, 0, read); + totalRead += read; + } + } + return totalRead; + } + + /** Retrieves committed block IDs from Azure Block Blob. */ + private List getCommittedBlockIds(BlockBlobClient blockBlobClient) { + List blockIds = new ArrayList<>(); + try { + BlockList blockList = blockBlobClient.listBlocks(BlockListType.COMMITTED); + if (blockList != null && blockList.getCommittedBlocks() != null) { + for (Block block : blockList.getCommittedBlocks()) { + blockIds.add(block.getName()); + } + } + } catch (Exception e) { + log.debug("Could not retrieve committed block list: {}", e.getMessage()); + } + return blockIds; + } + + /** Validates effective max append size limit. */ + private void validateMaxAppendSize(long totalAppended, Long effectiveMaxAppendSize) + throws MaxAppendSizeExceededException { + if (effectiveMaxAppendSize != null && totalAppended > effectiveMaxAppendSize) { + throw new MaxAppendSizeExceededException( + "Append size " + + totalAppended + + " exceeds maximum allowed chunk size of " + + effectiveMaxAppendSize); + } + } + + /** Validates min append size limit. */ + private void validateMinAppendSize(long totalAppended) throws MinAppendSizeNotMetException { + if (minAppendSize != null && totalAppended < minAppendSize) { + throw new MinAppendSizeNotMetException( + "Append size " + totalAppended + " is less than minimum allowed of " + minAppendSize); + } + } + + /** Stages pre-existing .part blob as a Block Blob block if present. */ + private void stagePartBlobIfPresent( + BlobClient partBlob, + long existingPartSize, + BlockBlobClient blockBlobClient, + List blockIds) + throws IOException { + if (existingPartSize > 0) { + String partBlockId = generateBlockId(blockIds.size()); + try (InputStream partIs = partBlob.openInputStream()) { + blockBlobClient.stageBlock(partBlockId, partIs, existingPartSize); + } + blockIds.add(partBlockId); + partBlob.deleteIfExists(); + } + } + + /** Stages local chunk temp file as a Block Blob block. */ + private void stageChunkFile( + File chunkFile, long chunkSize, BlockBlobClient blockBlobClient, List blockIds) + throws IOException { + if (chunkSize > 0) { + String chunkBlockId = generateBlockId(blockIds.size()); + try (InputStream chunkIs = new java.io.BufferedInputStream(new FileInputStream(chunkFile))) { + blockBlobClient.stageBlock(chunkBlockId, chunkIs, chunkSize); + } + blockIds.add(chunkBlockId); + } + } + + /** Processes remaining payload chunks from stream until EOF. */ + private long processRemainingChunks( + InputStream inputStream, + long optimalBlockSize, + Long effectiveMaxAppendSize, + UploadInfo upload, + BlobClient partBlob, + BlockBlobClient blockBlobClient, + List blockIds, + long currentTotalAppended) + throws IOException, MaxAppendSizeExceededException { + long additionalAppended = 0L; + boolean streamFinished = false; + + while (!streamFinished) { + File chunkFile = File.createTempFile("tus-azure-chunk-", ".tmp", tempBufferDir.toFile()); + try { + long chunkSize = readChunk(inputStream, chunkFile, optimalBlockSize); + if (chunkSize <= 0) { + break; + } + + additionalAppended += chunkSize; + long totalAppendedSoFar = currentTotalAppended + additionalAppended; + validateMaxAppendSize(totalAppendedSoFar, effectiveMaxAppendSize); + + long currentOffset = upload.getOffset() + totalAppendedSoFar; + boolean complete = upload.getLength() != null && currentOffset == upload.getLength(); + + if (chunkSize < optimalBlockSize && !complete) { + bufferToPartBlob(partBlob, 0L, chunkFile, chunkSize); + } else { + stageChunkFile(chunkFile, chunkSize, blockBlobClient, blockIds); + } + } finally { + deleteFileQuietly(chunkFile); + } + } + return additionalAppended; + } + + /** Buffers incoming data to temporary .part blob when under block threshold. */ + private void bufferToPartBlob( + BlobClient partBlob, long existingPartSize, File tempFile, long appendSize) + throws IOException { + if (existingPartSize == 0) { + partBlob.upload(BinaryData.fromFile(tempFile.toPath()), true); + } else { + File combinedTemp = File.createTempFile("tus-azure-part-", ".tmp", tempBufferDir.toFile()); + try { + try (InputStream partIs = partBlob.openInputStream(); + InputStream tempIs = new FileInputStream(tempFile); + SequenceInputStream seqIs = new SequenceInputStream(partIs, tempIs); + FileOutputStream fos = new FileOutputStream(combinedTemp)) { + IOUtils.copyLarge(seqIs, fos); + } + partBlob.upload(BinaryData.fromFile(combinedTemp.toPath()), true); + } finally { + deleteFileQuietly(combinedTemp); + } + } + } + + /** Commits empty block list for 0-byte upload completion. */ + private void commitEmptyDataBlob(UploadId id) { + BlockBlobClient blockBlobClient = + containerClient.getBlobClient(uploadPrefix + id).getBlockBlobClient(); + blockBlobClient.commitBlockList(new ArrayList<>(), true); + } + + /** Generates Base64 encoded block ID matching Azure Block Blob standards. */ + private String generateBlockId(int index) { + String idString = String.format("block-%06d", index); + return Base64.getEncoder().encodeToString(idString.getBytes(StandardCharsets.UTF_8)); + } + + /** Checks for deduplication match and links upload if duplicate. */ + private void checkAndApplyDeduplication(UploadInfo uploadInfo) throws IOException { + if (!isUploadDeduplicationEnabled() + || uploadInfo == null + || uploadInfo.getChecksum() == null + || uploadInfo.getChecksumAlgorithm() == null) { + return; + } + + String checksum = uploadInfo.getChecksum(); + ChecksumAlgorithm algorithm = uploadInfo.getChecksumAlgorithm(); + + UploadInfo existingUpload = getUploadInfoByChecksum(checksum, algorithm); + if (existingUpload != null && !existingUpload.getId().equals(uploadInfo.getId())) { + log.info( + "Found duplicate upload with checksum {}/{} (parent ID {})", + algorithm, + checksum, + existingUpload.getId()); + uploadInfo.setDuplicatesUploadId(existingUpload.getId()); + containerClient.getBlobClient(uploadPrefix + uploadInfo.getId()).deleteIfExists(); + } else { + String checksumKey = buildChecksumKey(checksum, algorithm); + BlobClient checksumBlob = containerClient.getBlobClient(checksumKey); + byte[] idBytes = uploadInfo.getId().toString().getBytes(StandardCharsets.UTF_8); + checksumBlob.upload(BinaryData.fromBytes(idBytes), true); + } + } + + /** Builds checksum index key path. */ + private String buildChecksumKey(String checksum, ChecksumAlgorithm algorithm) { + String algorithmName = algorithm != null ? algorithm.getTusName().toLowerCase() : "unknown"; + return checksumsPrefix + algorithmName + "/" + checksum; + } + + /** Sanitizes object key prefix format. */ + private String sanitizePrefix(String prefix) { + if (prefix == null || prefix.isEmpty()) { + return ""; + } + String result = prefix.startsWith("/") ? prefix.substring(1) : prefix; + return result.endsWith("/") ? result : result + "/"; + } + + /** Deletes local file suppressing exceptions. */ + private void deleteFileQuietly(File file) { + if (file != null && file.exists()) { + try { + Files.delete(file.toPath()); + } catch (Exception ignored) { + } + } + } + + /** Ensures local directory exists. */ + private void ensureDirectoryExists(Path dir) { + try { + if (!Files.exists(dir)) { + Files.createDirectories(dir); + } + } catch (IOException e) { + throw new RuntimeException("Could not create buffer directory " + dir, e); + } + } +} diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java new file mode 100644 index 00000000..76970e98 --- /dev/null +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java @@ -0,0 +1,115 @@ +package me.desair.tus.server.upload.azure; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.specialized.BlobLeaseClient; +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import me.desair.tus.server.upload.UploadLock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Distributed upload lock implementation backed by Azure Blob Storage Leases. + * + *

Azure Blob Leases provide native, atomic distributed locks. This class wraps an active lease + * and maintains a background daemon executor that periodically renews the lease (every 10 seconds + * for a standard 30-second lease) to prevent lock expiry during long-running streaming upload + * operations. + */ +public class AzureBlobUploadLock implements UploadLock { + + private static final Logger log = LoggerFactory.getLogger(AzureBlobUploadLock.class); + + private static final long RENEWAL_INTERVAL_SECONDS = 10L; + + private final BlobLeaseClient leaseClient; + private final BlobClient lockBlob; + private final String uploadUri; + private final ScheduledExecutorService renewalExecutor; + private volatile boolean released = false; + + /** + * Constructs an {@link AzureBlobUploadLock} wrapping an acquired Azure Blob lease. + * + * @param leaseClient The pre-acquired {@link BlobLeaseClient} holding the lease + * @param lockBlob The target lock {@link BlobClient} + * @param uploadUri The upload URI associated with this lock + */ + public AzureBlobUploadLock(BlobLeaseClient leaseClient, BlobClient lockBlob, String uploadUri) { + this.leaseClient = Objects.requireNonNull(leaseClient, "leaseClient must not be null"); + this.lockBlob = Objects.requireNonNull(lockBlob, "lockBlob must not be null"); + this.uploadUri = Objects.requireNonNull(uploadUri, "uploadUri must not be null"); + + // Initialize background daemon thread to renew lease periodically during upload + this.renewalExecutor = + Executors.newSingleThreadScheduledExecutor( + runnable -> { + Thread thread = new Thread(runnable, "azure-lease-renewal-" + uploadUri); + thread.setDaemon(true); + return thread; + }); + + scheduleLeaseRenewal(); + } + + /** Schedules periodic background renewal of the active lease. */ + private void scheduleLeaseRenewal() { + renewalExecutor.scheduleAtFixedRate( + this::renewLease, RENEWAL_INTERVAL_SECONDS, RENEWAL_INTERVAL_SECONDS, TimeUnit.SECONDS); + } + + /** Attempts to renew the lease with Azure Blob Storage. */ + private void renewLease() { + if (released) { + return; + } + try { + leaseClient.renewLease(); + log.trace("Successfully renewed Azure blob lease for upload URI {}", uploadUri); + } catch (Exception e) { + log.warn("Failed to renew Azure blob lease for upload URI {}: {}", uploadUri, e.getMessage()); + // ponytail: lease was broken externally or expired, shutdown executor + released = true; + shutdownExecutor(); + } + } + + @Override + public void release() { + if (!released) { + released = true; + shutdownExecutor(); + try { + leaseClient.releaseLease(); + log.trace("Released Azure blob lease for upload URI {}", uploadUri); + } catch (Exception e) { + log.debug( + "Azure blob lease release failed (may have already expired/broken) for URI {}: {}", + uploadUri, + e.getMessage()); + } + } + } + + @Override + public void close() throws IOException { + release(); + } + + @Override + public String getUploadUri() { + return uploadUri; + } + + /** Shuts down the renewal executor cleanly. */ + private void shutdownExecutor() { + try { + renewalExecutor.shutdownNow(); + } catch (Exception ignored) { + // Ignore shutdown interrupts + } + } +} diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureErrorType.java b/src/main/java/me/desair/tus/server/upload/azure/AzureErrorType.java new file mode 100644 index 00000000..002d0d35 --- /dev/null +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureErrorType.java @@ -0,0 +1,33 @@ +package me.desair.tus.server.upload.azure; + +import com.azure.storage.blob.models.BlobStorageException; + +/** Standardized Azure Blob Storage error types parsed from {@link BlobStorageException}. */ +public enum AzureErrorType { + /** Target blob or container does not exist (HTTP 404 / BlobNotFound / ContainerNotFound). */ + BLOB_NOT_FOUND, + + /** + * Blob lease is already held by another client / lock contention (HTTP 409 / LeaseAlreadyPresent + * / LeaseIdMismatchWithLeaseOperation). + */ + LEASE_ALREADY_PRESENT, + + /** No active lease exists on the blob (HTTP 409 / LeaseNotPresentWithLeaseOperation). */ + LEASE_NOT_PRESENT, + + /** Conditional request precondition failed (HTTP 412 / ConditionNotMet). */ + PRECONDITION_FAILED, + + /** Operation or API method not implemented by server/emulator (HTTP 501 / APINotImplemented). */ + API_NOT_IMPLEMENTED, + + /** Permission denied or invalid SAS credentials (HTTP 403 / AuthorizationFailure). */ + ACCESS_DENIED, + + /** Lock or resource already exists / general conflict (HTTP 409 / BlobAlreadyExists). */ + CONFLICT, + + /** Unknown or unmapped Azure storage exception. */ + UNKNOWN +} diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java b/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java new file mode 100644 index 00000000..b7e1a105 --- /dev/null +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java @@ -0,0 +1,67 @@ +package me.desair.tus.server.upload.azure; + +import com.azure.storage.blob.models.BlobStorageException; + +/** Utility helper methods for parsing and evaluating Azure Blob Storage error responses. */ +public final class AzureUtils { + + private AzureUtils() { + // Utility class + } + + /** + * Parses a {@link BlobStorageException} into a strongly-typed {@link AzureErrorType}. + * + * @param exception The Azure BlobStorageException to evaluate + * @return The corresponding AzureErrorType enum + */ + public static AzureErrorType parseErrorResponse(BlobStorageException exception) { + if (exception == null) { + return AzureErrorType.UNKNOWN; + } + + int statusCode = exception.getStatusCode(); + String rawErrorCode = ""; + try { + if (exception.getErrorCode() != null) { + rawErrorCode = exception.getErrorCode().toString(); + } + } catch (Exception ignored) { + // Defensive fallback if exception lacks initialized HTTP headers + } + + String errorCodeStr = rawErrorCode.replaceAll("[^a-zA-Z]", "").toLowerCase(); + + if (statusCode == 404 + || errorCodeStr.contains("blobnotfound") + || errorCodeStr.contains("containernotfound")) { + return AzureErrorType.BLOB_NOT_FOUND; + } + + if (statusCode == 409) { + if (errorCodeStr.contains("leasealreadypresent") + || errorCodeStr.contains("leaseidmismatchwithleaseoperation")) { + return AzureErrorType.LEASE_ALREADY_PRESENT; + } + if (errorCodeStr.contains("leasenotpresentwithleaseoperation") + || errorCodeStr.contains("leaseidmissing")) { + return AzureErrorType.LEASE_NOT_PRESENT; + } + return AzureErrorType.CONFLICT; + } + + if (statusCode == 412 || errorCodeStr.contains("conditionnotmet")) { + return AzureErrorType.PRECONDITION_FAILED; + } + + if (statusCode == 501 || errorCodeStr.contains("apinotimplemented")) { + return AzureErrorType.API_NOT_IMPLEMENTED; + } + + if (statusCode == 403 || errorCodeStr.contains("authorizationfailure")) { + return AzureErrorType.ACCESS_DENIED; + } + + return AzureErrorType.UNKNOWN; + } +} diff --git a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java index 30de697d..790c68f6 100644 --- a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java @@ -260,6 +260,13 @@ public void requestLockRelease(String requestUri) { lockingServiceDelegate.requestLockRelease(requestUri); } + @Override + public void close() throws IOException { + lockingServiceDelegate.close(); + storageServiceDelegate.close(); + cleanupCache(); + } + private void cleanupCache() { WeakReference ref = uploadInfoCache.get(); if (ref != null) { diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java b/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java index f702c1ac..96e12d15 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java @@ -1,5 +1,6 @@ package me.desair.tus.server.upload.disk; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -29,7 +30,8 @@ * File locks are also automatically released on application (JVM) shutdown. This means the file * locking is not persistent and prevents cleanup and stale lock issues. */ -public class DiskLockingService extends AbstractDiskBasedService implements UploadLockingService { +public class DiskLockingService extends AbstractDiskBasedService + implements UploadLockingService, Closeable { private static final Logger log = LoggerFactory.getLogger(DiskLockingService.class); private static final String LOCK_SUB_DIRECTORY = "locks"; @@ -41,10 +43,15 @@ public class DiskLockingService extends AbstractDiskBasedService implements Uplo private static Thread watchdogThread = null; private static final Object watchdogLock = new Object(); + private final Thread shutdownHook; + private volatile boolean closed = false; + private UploadIdFactory idFactory; public DiskLockingService(String storagePath) { super(storagePath + File.separator + LOCK_SUB_DIRECTORY); + this.shutdownHook = new Thread(this::closeQuietly, "disk-lock-shutdown-hook"); + registerShutdownHook(); } /** Constructor to use custom UploadIdFactory. */ @@ -54,6 +61,44 @@ public DiskLockingService(UploadIdFactory idFactory, String storagePath) { this.idFactory = idFactory; } + private void registerShutdownHook() { + try { + Runtime.getRuntime().addShutdownHook(shutdownHook); + } catch (IllegalStateException ignored) { + // JVM is already shutting down + } + } + + private void deregisterShutdownHook() { + try { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } catch (IllegalStateException ignored) { + // JVM is already shutting down + } + } + + private void closeQuietly() { + try { + close(); + } catch (Exception ignored) { + } + } + + @Override + public void close() throws IOException { + if (!closed) { + closed = true; + deregisterShutdownHook(); + synchronized (watchdogLock) { + if (watchdogThread != null) { + watchdogThread.interrupt(); + watchdogThread = null; + } + } + activeLocks.clear(); + } + } + /** * Attempts to lock the upload resource. Wraps the lock in a RegisteredLock decorator to manage * cleanup of stop files and the active lock registry. diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java index 9d170b64..f798de51 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java @@ -169,21 +169,24 @@ public void merge(UploadInfo uploadInfo) throws IOException, UploadNotFoundExcep @Override public InputStream getConcatenatedBytes(UploadInfo uploadInfo) throws IOException, UploadNotFoundException { - if (uploadInfo == null) { return null; } - if (uploadInfo.getStorageUploadId() == null) { + if (uploadStorageService == null) { + throw new IOException( + "UploadStorageService must be configured to retrieve concatenated upload bytes"); + } + + if (uploadInfo.isUploadInProgress()) { merge(uploadInfo); } - if (uploadStorageService != null) { + if (!uploadInfo.isUploadInProgress()) { return uploadStorageService.getUploadedBytes(uploadInfo.getId()); } - throw new IOException( - "UploadStorageService must be configured to retrieve concatenated upload bytes"); + return new java.io.ByteArrayInputStream(new byte[0]); } @Override diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ErrorType.java b/src/main/java/me/desair/tus/server/upload/s3/S3ErrorType.java index 11b82f77..901b9039 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3ErrorType.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ErrorType.java @@ -16,6 +16,9 @@ public enum S3ErrorType { /** Permission denied or invalid credentials (HTTP 403 / AccessDenied). */ ACCESS_DENIED, + /** Operation or API method not implemented by server/emulator (HTTP 501 / APINotImplemented). */ + API_NOT_IMPLEMENTED, + /** Unknown or unmapped S3 error response. */ UNKNOWN } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index 823f8db0..e7d857a4 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -10,6 +10,7 @@ import io.minio.errors.ErrorResponseException; import io.minio.messages.Item; import java.io.ByteArrayInputStream; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.util.Map; @@ -54,7 +55,7 @@ * pods. * */ -public class S3LockingService implements UploadLockingService { +public class S3LockingService implements UploadLockingService, Closeable { private static final Logger log = LoggerFactory.getLogger(S3LockingService.class); @@ -72,6 +73,9 @@ public class S3LockingService implements UploadLockingService { private final Map activeInputStreams = new ConcurrentHashMap<>(); private final ScheduledExecutorService watchdogExecutor; + private final Thread shutdownHook; + private volatile boolean closed = false; + /** * Basic constructor using default lock prefix ("locks/"), 30s lease duration, and 2s polling * interval. @@ -122,6 +126,33 @@ public S3LockingService( this.watchdogExecutor.scheduleAtFixedRate( this::checkStopSignals, pollIntervalMs, pollIntervalMs, TimeUnit.MILLISECONDS); } + + // Register automatic JVM shutdown hook to clean up watchdog thread pool on app/pod shutdown + this.shutdownHook = new Thread(this::closeQuietly, "s3-lock-shutdown-hook"); + registerShutdownHook(); + } + + private void registerShutdownHook() { + try { + Runtime.getRuntime().addShutdownHook(shutdownHook); + } catch (IllegalStateException ignored) { + // JVM is already shutting down + } + } + + private void deregisterShutdownHook() { + try { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } catch (IllegalStateException ignored) { + // JVM is already shutting down + } + } + + private void closeQuietly() { + try { + close(); + } catch (Exception ignored) { + } } @Override @@ -286,6 +317,19 @@ private void writeStopSignal(UploadId uploadId) { } } + @Override + public void close() throws IOException { + if (!closed) { + closed = true; + deregisterShutdownHook(); + try { + watchdogExecutor.shutdownNow(); + } catch (Exception ignored) { + } + activeInputStreams.clear(); + } + } + private void checkStopSignals() { for (Map.Entry entry : activeInputStreams.entrySet()) { checkStopSignalForEntry(entry.getKey(), entry.getValue()); @@ -305,7 +349,7 @@ private void checkStopSignalForEntry(String uri, InputStream inputStream) { // Remote stop signal object found! Interrupt local byte stream immediately interruptStream(inputStream); } catch (ErrorResponseException e) { - if ("NoSuchKey".equalsIgnoreCase(e.errorResponse().code())) { + if (S3Utils.parseErrorResponse(e) == S3ErrorType.NO_SUCH_KEY) { // Normal state: no stop signal object in S3 return; } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index 5a41b88d..8045d029 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -172,6 +172,11 @@ public String getS3ObjectKey(UploadInfo uploadInfo) { return null; } + // Resolve duplicate child upload dynamically to parent S3 object key per AGENTS.md §7 + if (uploadInfo.getDuplicatesUploadId() != null) { + return buildObjectKey(uploadInfo.getDuplicatesUploadId()); + } + if (uploadInfo.getStorageUploadId() != null) { return uploadInfo.getStorageUploadId(); } else { @@ -372,7 +377,7 @@ public InputStream getUploadedBytes(UploadId id) throws IOException, UploadNotFo } // Handle concatenated upload resolution if applicable - if (UploadType.CONCATENATED.equals(info.getUploadType()) && info.getStorageUploadId() == null) { + if (UploadType.CONCATENATED.equals(info.getUploadType()) && info.isUploadInProgress()) { if (concatenationService != null) { concatenationService.merge(info); info = getUploadInfo(id); @@ -466,6 +471,10 @@ public void terminateUpload(UploadInfo uploadInfo) throws UploadNotFoundExceptio deleteObjectQuietly( buildChecksumKey(uploadInfo.getChecksum(), uploadInfo.getChecksumAlgorithm())); } + + // Delete lock target and stop signal objects + deleteObjectQuietly(buildLockKey(uploadInfo.getId())); + deleteObjectQuietly(buildStopKey(uploadInfo.getId())); } @Override @@ -630,7 +639,7 @@ private InputStream prepareStreamWithExistingIncompletePart( return new SequenceInputStream(new FileInputStream(tempPrependedFile), inputStream); } } catch (ErrorResponseException e) { - if ("NoSuchKey".equalsIgnoreCase(e.errorResponse().code())) { + if (S3Utils.parseErrorResponse(e) == S3ErrorType.NO_SUCH_KEY) { // Normal case: no leftover .part object present in S3 } } catch (Exception ignored) { @@ -1016,6 +1025,14 @@ private String buildChecksumKey(String checksum, ChecksumAlgorithm algorithm) { return checksumsPrefix + algorithmName + "/" + checksum; } + private String buildLockKey(UploadId id) { + return locksPrefix + id.toString() + ".lock"; + } + + private String buildStopKey(UploadId id) { + return locksPrefix + id.toString() + ".stop"; + } + private static class AppendResult { final long totalBytesAppended; final List allPartKeys; diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java b/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java index 675971e7..618f41c4 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java @@ -22,9 +22,11 @@ public static S3ErrorType parseErrorResponse(ErrorResponseException exception) { } ErrorResponse response = exception.errorResponse(); - String code = response != null ? response.code() : ""; + String code = response != null && response.code() != null ? response.code() : ""; - if ("NoSuchKey".equalsIgnoreCase(code) || "NoSuchBucket".equalsIgnoreCase(code)) { + if ("NoSuchKey".equalsIgnoreCase(code) + || "NoSuchBucket".equalsIgnoreCase(code) + || "NoSuchUpload".equalsIgnoreCase(code)) { return S3ErrorType.NO_SUCH_KEY; } @@ -41,6 +43,10 @@ public static S3ErrorType parseErrorResponse(ErrorResponseException exception) { return S3ErrorType.ACCESS_DENIED; } + if ("APINotImplemented".equalsIgnoreCase(code) || "NotImplemented".equalsIgnoreCase(code)) { + return S3ErrorType.API_NOT_IMPLEMENTED; + } + return S3ErrorType.UNKNOWN; } } diff --git a/src/test/java/me/desair/tus/server/TestUtils.java b/src/test/java/me/desair/tus/server/TestUtils.java index 7a48b210..db0b1bf1 100644 --- a/src/test/java/me/desair/tus/server/TestUtils.java +++ b/src/test/java/me/desair/tus/server/TestUtils.java @@ -104,4 +104,47 @@ public static void createBucket(MinioClient minioClient, String bucket) { throw new RuntimeException("Failed to create bucket " + bucket, e); } } + + /** + * Create and configure a GenericContainer running Azurite for Azure integration testing. + * + * @return A configured GenericContainer instance (not started yet) + */ + public static GenericContainer createAzuriteContainer() { + return new GenericContainer<>("mcr.microsoft.com/azure-storage/azurite:latest") + .withExposedPorts(10000) + .withCommand("azurite-blob", "--blobHost", "0.0.0.0", "--skipApiVersionCheck"); + } + + /** + * Create a {@link com.azure.storage.blob.BlobContainerClient} connected to Azurite Testcontainer. + * + * @param azurite The active Azurite Testcontainer + * @param containerName Target container name + * @return Pre-configured BlobContainerClient + */ + public static com.azure.storage.blob.BlobContainerClient createBlobContainerClient( + GenericContainer azurite, String containerName) { + String connectionString = + String.format( + "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" + + "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" + + "K1SZFPTOtr/KBHBeksoGMGw==;" + + "BlobEndpoint=http://%s:%d/devstoreaccount1", + azurite.getHost(), azurite.getMappedPort(10000)); + + com.azure.storage.blob.BlobServiceClient serviceClient = + new com.azure.storage.blob.BlobServiceClientBuilder() + .connectionString(connectionString) + .buildClient(); + + com.azure.storage.blob.BlobContainerClient containerClient = + serviceClient.getBlobContainerClient(containerName); + + if (!containerClient.exists()) { + containerClient.create(); + } + + return containerClient; + } } diff --git a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java index 55a2eee3..8d1cb1ce 100644 --- a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java +++ b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java @@ -522,4 +522,15 @@ public void testWithJsonSerialization() { service.withJsonSerialization(false); org.junit.Assert.assertFalse(service.getUploadStorageService().isJsonSerializationEnabled()); } + + @Test + public void testClose() throws Exception { + UploadLockingService mockLockingService = mock(UploadLockingService.class); + TusFileUploadService service = + new TusFileUploadService().withUploadLockingService(mockLockingService); + + service.close(); + + verify(mockLockingService).close(); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java new file mode 100644 index 00000000..e0c76398 --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java @@ -0,0 +1,135 @@ +package me.desair.tus.server.upload.azure; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.azure.storage.blob.BlobContainerClient; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.Arrays; +import java.util.List; +import me.desair.tus.server.TestUtils; +import me.desair.tus.server.exception.UploadNotFoundException; +import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadType; +import org.junit.Assume; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.testcontainers.containers.GenericContainer; + +public class AzureBlobConcatenationServiceTest { + + @ClassRule + public static GenericContainer azuriteContainer = TestUtils.createAzuriteContainer(); + + private BlobContainerClient containerClient; + private AzureBlobStorageService storageService; + private AzureBlobConcatenationService concatenationService; + + @Before + public void setUp() { + Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); + containerClient = + TestUtils.createBlobContainerClient( + azuriteContainer, "concat-unit-container-" + System.nanoTime()); + storageService = new AzureBlobStorageService(containerClient); + concatenationService = new AzureBlobConcatenationService(containerClient, storageService); + storageService.setUploadConcatenationService(concatenationService); + } + + @Test + public void mergeShouldDoNothingIfInfoIsNull() throws Exception { + concatenationService.merge(null); + } + + @Test + public void mergeShouldDoNothingIfConcatFilesIsNull() throws Exception { + UploadInfo info = new UploadInfo(); + concatenationService.merge(info); + } + + @Test(expected = UploadNotFoundException.class) + public void mergeShouldThrowWhenPartialUploadNotFound() throws Exception { + UploadInfo finalInfo = new UploadInfo(); + finalInfo.setId(new me.desair.tus.server.upload.UploadId("final-id")); + finalInfo.setConcatenationPartIds(Arrays.asList("/test/upload/non-existing")); + finalInfo.setUploadType(UploadType.CONCATENATED); + + concatenationService.merge(finalInfo); + } + + @Test + public void mergeShouldStageBlocksAndCommitBlockList() throws Exception { + UploadInfo part1Info = new UploadInfo(); + part1Info.setLength(10L); + UploadInfo part1 = storageService.create(part1Info, null); + storageService.append(part1, new ByteArrayInputStream("part1-data".getBytes())); + + UploadInfo part2Info = new UploadInfo(); + part2Info.setLength(10L); + UploadInfo part2 = storageService.create(part2Info, null); + storageService.append(part2, new ByteArrayInputStream("part2-data".getBytes())); + + UploadInfo finalInfo = new UploadInfo(); + finalInfo.setConcatenationPartIds( + Arrays.asList("/test/upload/" + part1.getId(), "/test/upload/" + part2.getId())); + finalInfo.setUploadType(UploadType.CONCATENATED); + UploadInfo createdFinal = storageService.create(finalInfo, null); + + concatenationService.merge(createdFinal); + + assertEquals(Long.valueOf(20L), createdFinal.getOffset()); + assertEquals(Long.valueOf(20L), createdFinal.getLength()); + } + + @Test + public void getConcatenationBytesShouldReturnCombinedStream() throws Exception { + UploadInfo part1Info = new UploadInfo(); + part1Info.setLength(5L); + UploadInfo part1 = storageService.create(part1Info, null); + storageService.append(part1, new ByteArrayInputStream("hello".getBytes())); + + UploadInfo part2Info = new UploadInfo(); + part2Info.setLength(6L); + UploadInfo part2 = storageService.create(part2Info, null); + storageService.append(part2, new ByteArrayInputStream("-world".getBytes())); + + UploadInfo finalInfo = new UploadInfo(); + finalInfo.setConcatenationPartIds( + Arrays.asList("/test/upload/" + part1.getId(), "/test/upload/" + part2.getId())); + finalInfo.setUploadType(UploadType.CONCATENATED); + UploadInfo createdFinal = storageService.create(finalInfo, null); + + InputStream is = concatenationService.getConcatenatedBytes(createdFinal); + assertNotNull(is); + assertEquals( + "hello-world", + org.apache.commons.io.IOUtils.toString(is, java.nio.charset.StandardCharsets.UTF_8)); + } + + @Test(expected = UploadNotFoundException.class) + public void getConcatenatedBytesShouldThrowOnNullInfo() throws Exception { + concatenationService.getConcatenatedBytes(null); + } + + @Test + public void getPartialUploadsShouldReturnList() throws Exception { + UploadInfo part1Info = new UploadInfo(); + part1Info.setLength(5L); + UploadInfo part1 = storageService.create(part1Info, null); + + UploadInfo finalInfo = new UploadInfo(); + finalInfo.setConcatenationPartIds(Arrays.asList("/test/upload/" + part1.getId())); + + List partials = concatenationService.getPartialUploads(finalInfo); + assertEquals(1, partials.size()); + assertEquals(part1.getId(), partials.get(0).getId()); + } + + @Test + public void getPartialUploadsShouldReturnEmptyOnNullInfo() throws Exception { + assertTrue(concatenationService.getPartialUploads(null).isEmpty()); + } +} diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java new file mode 100644 index 00000000..29fbe917 --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java @@ -0,0 +1,98 @@ +package me.desair.tus.server.upload.azure; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import com.azure.storage.blob.BlobContainerClient; +import java.io.ByteArrayInputStream; +import me.desair.tus.server.TestUtils; +import me.desair.tus.server.exception.UploadAlreadyLockedException; +import me.desair.tus.server.upload.TimeBasedUploadIdFactory; +import me.desair.tus.server.upload.UploadId; +import me.desair.tus.server.upload.UploadLock; +import me.desair.tus.server.util.InterruptibleInputStream; +import org.junit.Assume; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.testcontainers.containers.GenericContainer; + +public class AzureBlobLockingServiceTest { + + @ClassRule + public static GenericContainer azuriteContainer = TestUtils.createAzuriteContainer(); + + private BlobContainerClient containerClient; + private AzureBlobLockingService lockingService; + + @Before + public void setUp() { + Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); + containerClient = + TestUtils.createBlobContainerClient( + azuriteContainer, "lock-unit-container-" + System.nanoTime()); + lockingService = new AzureBlobLockingService(containerClient); + TimeBasedUploadIdFactory idFactory = new TimeBasedUploadIdFactory(); + idFactory.setUploadUri("/test/upload"); + lockingService.setIdFactory(idFactory); + } + + @Test + public void lockUploadByUriShouldReturnNullOnInvalidUri() throws Exception { + assertNull(lockingService.lockUploadByUri("invalid-uri-no-id")); + } + + @Test + public void isLockedShouldReturnFalseWhenLockBlobDoesNotExist() { + assertFalse(lockingService.isLocked(new UploadId("12345"))); + } + + @Test + public void lockUploadByUriShouldAcquireLock() throws Exception { + UploadLock lock = lockingService.lockUploadByUri("/test/upload/12345"); + assertNotNull(lock); + assertEquals("/test/upload/12345", lock.getUploadUri()); + assertTrue(lockingService.isLocked(new UploadId("12345"))); + lock.release(); + } + + @Test(expected = UploadAlreadyLockedException.class) + public void lockUploadByUriShouldThrowOnLockContention() throws Exception { + UploadLock lock1 = lockingService.lockUploadByUri("/test/upload/12345"); + assertNotNull(lock1); + try { + // Second lock attempt on same URI should throw UploadAlreadyLockedException + lockingService.lockUploadByUri("/test/upload/12345"); + } finally { + lock1.release(); + } + } + + @Test + public void registerInputStreamAndRequestReleaseShouldInterruptStream() throws Exception { + ByteArrayInputStream bais = new ByteArrayInputStream("data".getBytes()); + InterruptibleInputStream stream = new InterruptibleInputStream(bais); + + lockingService.registerInputStream("/test/upload/12345", stream); + lockingService.requestLockRelease("/test/upload/12345"); + + try { + stream.read(); + } catch (Exception e) { + assertNotNull(e); + } + } + + @Test + public void cleanupStaleLocksShouldNotThrow() throws Exception { + lockingService.cleanupStaleLocks(); + } + + @Test + public void closeShouldCleanUpResources() throws Exception { + lockingService.close(); + } +} diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java new file mode 100644 index 00000000..d265021f --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java @@ -0,0 +1,179 @@ +package me.desair.tus.server.upload.azure; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import com.azure.storage.blob.BlobContainerClient; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import me.desair.tus.server.TestUtils; +import me.desair.tus.server.checksum.ChecksumAlgorithm; +import me.desair.tus.server.exception.MaxAppendSizeExceededException; +import me.desair.tus.server.upload.UploadId; +import me.desair.tus.server.upload.UploadInfo; +import org.junit.Assume; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.testcontainers.containers.GenericContainer; + +public class AzureBlobStorageServiceTest { + + @ClassRule + public static GenericContainer azuriteContainer = TestUtils.createAzuriteContainer(); + + private BlobContainerClient containerClient; + private AzureBlobStorageService storageService; + + @Before + public void setUp() { + Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); + containerClient = + TestUtils.createBlobContainerClient( + azuriteContainer, "unit-test-container-" + System.nanoTime()); + storageService = new AzureBlobStorageService(containerClient); + } + + @Test + public void createShouldSetIdAndSaveInfo() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(1000L); + + UploadInfo created = storageService.create(info, "owner1"); + + assertNotNull(created.getId()); + assertEquals("owner1", created.getOwnerKey()); + assertEquals(Long.valueOf(0L), created.getOffset()); + assertEquals("uploads/" + created.getId(), created.getStorageUploadId()); + } + + @Test + public void createZeroLengthUploadShouldCommitEmptyDataBlob() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(0L); + + UploadInfo created = storageService.create(info, "owner1"); + + assertNotNull(created.getId()); + assertEquals(Long.valueOf(0L), created.getOffset()); + } + + @Test + public void getUploadInfoShouldReturnInfo() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(500L); + UploadInfo created = storageService.create(info, "owner1"); + + UploadInfo fetched = storageService.getUploadInfo(created.getId()); + assertNotNull(fetched); + assertEquals(created.getId(), fetched.getId()); + assertEquals(Long.valueOf(500L), fetched.getLength()); + } + + @Test + public void getUploadInfoNotFoundShouldReturnNull() throws Exception { + UploadInfo fetched = storageService.getUploadInfo(new UploadId("non-existing-id")); + assertNull(fetched); + } + + @Test + public void getUploadInfoOwnerIsolation() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(500L); + UploadInfo created = storageService.create(info, "owner1"); + + String url = "/test/upload/" + created.getId(); + assertNull(storageService.getUploadInfo(url, "wrong-owner")); + assertNotNull(storageService.getUploadInfo(url, "owner1")); + } + + @Test + public void maxAppendSizeFallback() { + storageService.setMaxUploadSize(5000L); + assertEquals(Long.valueOf(5000L), storageService.getMaxAppendSize()); + + storageService.setMaxAppendSize(2000L); + assertEquals(Long.valueOf(2000L), storageService.getMaxAppendSize()); + } + + @Test(expected = MaxAppendSizeExceededException.class) + public void appendExceedsMaxAppendSizeShouldThrow() throws Exception { + storageService.setMaxAppendSize(10L); + + UploadInfo info = new UploadInfo(); + info.setLength(100L); + UploadInfo created = storageService.create(info, "owner1"); + + ByteArrayInputStream bais = new ByteArrayInputStream("01234567890123456789".getBytes()); + storageService.append(created, bais); + } + + @Test + public void terminateUploadShouldDeleteBlobs() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(1000L); + info.setChecksum("5d41402abc4b2a76b9719d911017c592"); + info.setChecksumAlgorithm(ChecksumAlgorithm.MD5); + + UploadInfo created = storageService.create(info, "owner1"); + + storageService.terminateUpload(created); + assertNull(storageService.getUploadInfo(created.getId())); + } + + @Test + public void getAzureBlobNameShouldReturnBlobName() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(1000L); + UploadInfo created = storageService.create(info, "owner1"); + + String blobName = storageService.getAzureBlobName("/test/upload/" + created.getId(), "owner1"); + assertEquals("uploads/" + created.getId(), blobName); + } + + @Test + public void copyUploadToShouldCopyDataToOutputStream() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(9L); + UploadInfo created = storageService.create(info, "owner1"); + + ByteArrayInputStream bais = new ByteArrayInputStream("test-data".getBytes()); + storageService.append(created, bais); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + storageService.copyUploadTo(created, baos); + + assertEquals("test-data", baos.toString()); + } + + @Test + public void getUploadedBytesShouldReturnInputStream() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(9L); + UploadInfo created = storageService.create(info, "owner1"); + + ByteArrayInputStream bais = new ByteArrayInputStream("test-data".getBytes()); + storageService.append(created, bais); + + try (java.io.InputStream is = storageService.getUploadedBytes(created.getId())) { + assertNotNull(is); + assertEquals( + "test-data", + org.apache.commons.io.IOUtils.toString(is, java.nio.charset.StandardCharsets.UTF_8)); + } + } + + @Test + public void removeLastNumberOfBytesPartBlobOnly() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(10L); + UploadInfo created = storageService.create(info, "owner1"); + + storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + assertEquals(Long.valueOf(10L), created.getOffset()); + + storageService.removeLastNumberOfBytes(created, 3L); + assertEquals(Long.valueOf(7L), created.getOffset()); + } +} diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java new file mode 100644 index 00000000..c497e416 --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java @@ -0,0 +1,62 @@ +package me.desair.tus.server.upload.azure; + +import static org.junit.Assert.assertEquals; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.specialized.BlobLeaseClient; +import com.azure.storage.blob.specialized.BlobLeaseClientBuilder; +import me.desair.tus.server.TestUtils; +import me.desair.tus.server.upload.UploadLock; +import org.junit.Assume; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.testcontainers.containers.GenericContainer; + +public class AzureBlobUploadLockTest { + + @ClassRule + public static GenericContainer azuriteContainer = TestUtils.createAzuriteContainer(); + + private BlobContainerClient containerClient; + private String uploadUri; + private UploadLock uploadLock; + + @Before + public void setUp() throws Exception { + Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); + containerClient = + TestUtils.createBlobContainerClient( + azuriteContainer, "lock-unit-container2-" + System.nanoTime()); + + uploadUri = "/test/upload/12345"; + BlobClient lockBlob = containerClient.getBlobClient("locks/12345.lock"); + lockBlob.getAppendBlobClient().create(); + BlobLeaseClient leaseClient = new BlobLeaseClientBuilder().blobClient(lockBlob).buildClient(); + leaseClient.acquireLease(30); + + uploadLock = new AzureBlobUploadLock(leaseClient, lockBlob, uploadUri); + } + + @Test + public void getUploadUriShouldReturnUri() { + assertEquals(uploadUri, uploadLock.getUploadUri()); + } + + @Test + public void releaseShouldReleaseLease() { + uploadLock.release(); + } + + @Test + public void closeShouldCallRelease() throws Exception { + uploadLock.close(); + } + + @Test + public void doubleReleaseShouldBeIdempotent() { + uploadLock.release(); + uploadLock.release(); + } +} diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java new file mode 100644 index 00000000..b259c0e3 --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java @@ -0,0 +1,79 @@ +package me.desair.tus.server.upload.azure; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpResponse; +import com.azure.storage.blob.models.BlobErrorCode; +import com.azure.storage.blob.models.BlobStorageException; +import org.junit.Test; + +public class AzureUtilsTest { + + private BlobStorageException createException(int statusCode, BlobErrorCode errorCode) { + HttpResponse response = mock(HttpResponse.class); + when(response.getStatusCode()).thenReturn(statusCode); + HttpHeaders headers = new HttpHeaders(); + if (errorCode != null) { + headers.set("x-ms-error-code", errorCode.toString()); + } + when(response.getHeaders()).thenReturn(headers); + return new BlobStorageException("Test exception", response, errorCode); + } + + @Test + public void testParseErrorResponseNull() { + assertEquals(AzureErrorType.UNKNOWN, AzureUtils.parseErrorResponse(null)); + } + + @Test + public void testParseErrorResponseBlobNotFound() { + BlobStorageException ex = createException(404, BlobErrorCode.BLOB_NOT_FOUND); + assertEquals(AzureErrorType.BLOB_NOT_FOUND, AzureUtils.parseErrorResponse(ex)); + } + + @Test + public void testParseErrorResponseLeaseAlreadyPresent() { + BlobStorageException ex = createException(409, BlobErrorCode.LEASE_ALREADY_PRESENT); + assertEquals(AzureErrorType.LEASE_ALREADY_PRESENT, AzureUtils.parseErrorResponse(ex)); + } + + @Test + public void testParseErrorResponseLeaseNotPresent() { + BlobStorageException ex = + createException(409, BlobErrorCode.LEASE_NOT_PRESENT_WITH_LEASE_OPERATION); + assertEquals(AzureErrorType.LEASE_NOT_PRESENT, AzureUtils.parseErrorResponse(ex)); + } + + @Test + public void testParseErrorResponseConflict() { + BlobStorageException ex = createException(409, BlobErrorCode.BLOB_ALREADY_EXISTS); + assertEquals(AzureErrorType.CONFLICT, AzureUtils.parseErrorResponse(ex)); + } + + @Test + public void testParseErrorResponsePreconditionFailed() { + BlobStorageException ex = createException(412, BlobErrorCode.CONDITION_NOT_MET); + assertEquals(AzureErrorType.PRECONDITION_FAILED, AzureUtils.parseErrorResponse(ex)); + } + + @Test + public void testParseErrorResponseApiNotImplemented() { + BlobStorageException ex = createException(501, null); + assertEquals(AzureErrorType.API_NOT_IMPLEMENTED, AzureUtils.parseErrorResponse(ex)); + } + + @Test + public void testParseErrorResponseAccessDenied() { + BlobStorageException ex = createException(403, BlobErrorCode.AUTHORIZATION_FAILURE); + assertEquals(AzureErrorType.ACCESS_DENIED, AzureUtils.parseErrorResponse(ex)); + } + + @Test + public void testParseErrorResponseUnknown() { + BlobStorageException ex = createException(500, null); + assertEquals(AzureErrorType.UNKNOWN, AzureUtils.parseErrorResponse(ex)); + } +} diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobRufhProtocol.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobRufhProtocol.java new file mode 100644 index 00000000..02a9d47d --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobRufhProtocol.java @@ -0,0 +1,59 @@ +package me.desair.tus.server.upload.azure; + +import com.azure.storage.blob.BlobContainerClient; +import me.desair.tus.server.AbstractITRufhProtocol; +import me.desair.tus.server.TestUtils; +import me.desair.tus.server.TusFileUploadService; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.testcontainers.containers.GenericContainer; + +/** + * End-to-end integration test suite verifying the IETF Resumable Uploads for HTTP (RUFH) protocol + * implementation backed by {@link AzureBlobStorageService} and {@link AzureBlobLockingService} on + * Azurite using the official Azure Storage Blob SDK. + */ +public class ITAzureBlobRufhProtocol extends AbstractITRufhProtocol { + + private static GenericContainer azurite; + private static BlobContainerClient containerClient; + private static final String CONTAINER = "test-rufh-azure-container"; + + @BeforeClass + public static void setUpClass() { + org.junit.Assume.assumeTrue( + "Container runtime is not available; skipping Testcontainers Azurite test", + TestUtils.isContainerRuntimeAvailable()); + + azurite = TestUtils.createAzuriteContainer(); + azurite.start(); + + containerClient = TestUtils.createBlobContainerClient(azurite, CONTAINER); + } + + @AfterClass + public static void tearDownClass() { + if (azurite != null) { + azurite.stop(); + } + } + + @Override + protected TusFileUploadService createTusFileUploadService() { + org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); + + AzureBlobStorageService azureStorage = new AzureBlobStorageService(containerClient); + AzureBlobLockingService azureLocking = new AzureBlobLockingService(containerClient); + AzureBlobConcatenationService azureConcat = + new AzureBlobConcatenationService(containerClient, azureStorage); + azureStorage.setUploadConcatenationService(azureConcat); + + return new TusFileUploadService() + .withUploadUri(UPLOAD_URI) + .withUploadStorageService(azureStorage) + .withUploadLockingService(azureLocking) + .withMaxUploadSize(1073741824L) + .withUploadExpirationPeriod(2L * 24 * 60 * 60 * 1000) + .withDownloadFeature(); + } +} diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageServiceTest.java new file mode 100644 index 00000000..3f5de690 --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageServiceTest.java @@ -0,0 +1,128 @@ +package me.desair.tus.server.upload.azure; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import com.azure.storage.blob.BlobContainerClient; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import me.desair.tus.server.TestUtils; +import me.desair.tus.server.checksum.ChecksumAlgorithm; +import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLock; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.testcontainers.containers.GenericContainer; + +public class ITAzureBlobStorageServiceTest { + + private static GenericContainer azurite; + private static BlobContainerClient containerClient; + private static final String CONTAINER = "test-storage-azure-container"; + + @BeforeClass + public static void setUpClass() { + org.junit.Assume.assumeTrue( + "Container runtime is not available; skipping Testcontainers Azurite test", + TestUtils.isContainerRuntimeAvailable()); + + azurite = TestUtils.createAzuriteContainer(); + azurite.start(); + + containerClient = TestUtils.createBlobContainerClient(azurite, CONTAINER); + } + + @AfterClass + public static void tearDownClass() { + if (azurite != null) { + azurite.stop(); + } + } + + @Test + public void testFullUploadLifecycleOnAzurite() throws Exception { + org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); + + AzureBlobStorageService storage = new AzureBlobStorageService(containerClient); + AzureBlobLockingService locking = new AzureBlobLockingService(containerClient); + + UploadInfo info = new UploadInfo(); + info.setLength(11L); + + UploadInfo created = storage.create(info, "owner1"); + assertNotNull(created.getId()); + + try (UploadLock lock = locking.lockUploadByUri("/test/upload/" + created.getId())) { + assertNotNull(lock); + storage.append(created, new ByteArrayInputStream("hello ".getBytes())); + storage.append(created, new ByteArrayInputStream("world".getBytes())); + } + + UploadInfo fetched = storage.getUploadInfo(created.getId()); + assertNotNull(fetched); + assertEquals(Long.valueOf(11L), fetched.getOffset()); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + storage.copyUploadTo(fetched, baos); + assertEquals("hello world", baos.toString()); + + String blobName = storage.getAzureBlobName("/test/upload/" + created.getId(), "owner1"); + assertEquals("uploads/" + created.getId(), blobName); + } + + @Test + public void testTruncateBytesOnAzurite() throws Exception { + org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); + + AzureBlobStorageService storage = new AzureBlobStorageService(containerClient); + + UploadInfo info = new UploadInfo(); + info.setLength(10L); + UploadInfo created = storage.create(info, "owner1"); + + storage.append(created, new ByteArrayInputStream("0123456789".getBytes())); + assertEquals(Long.valueOf(10L), created.getOffset()); + + // Truncate 3 bytes + storage.removeLastNumberOfBytes(created, 3L); + assertEquals(Long.valueOf(7L), created.getOffset()); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + storage.copyUploadTo(created, baos); + assertEquals("0123456", baos.toString()); + } + + @Test + public void testDeduplicationOnAzurite() throws Exception { + org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); + + AzureBlobStorageService storage = new AzureBlobStorageService(containerClient); + storage.setUploadDeduplicationEnabled(true); + + // Parent upload + UploadInfo parent = new UploadInfo(); + parent.setLength(5L); + parent.setChecksum("5d41402abc4b2a76b9719d911017c592"); + parent.setChecksumAlgorithm(ChecksumAlgorithm.MD5); + parent = storage.create(parent, "owner1"); + storage.append(parent, new ByteArrayInputStream("hello".getBytes())); + + // Child upload matching parent checksum + UploadInfo child = new UploadInfo(); + child.setLength(5L); + child.setChecksum("5d41402abc4b2a76b9719d911017c592"); + child.setChecksumAlgorithm(ChecksumAlgorithm.MD5); + child = storage.create(child, "owner1"); + storage.append(child, new ByteArrayInputStream("hello".getBytes())); + + assertNotNull(child.getDuplicatesUploadId()); + assertEquals(parent.getId(), child.getDuplicatesUploadId()); + + // Clean up parent + storage.terminateUpload(parent); + assertNull( + storage.getUploadInfoByChecksum("5d41402abc4b2a76b9719d911017c592", ChecksumAlgorithm.MD5)); + } +} diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobTusFileUploadService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobTusFileUploadService.java new file mode 100644 index 00000000..6a9f9d52 --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobTusFileUploadService.java @@ -0,0 +1,68 @@ +package me.desair.tus.server.upload.azure; + +import com.azure.storage.blob.BlobContainerClient; +import me.desair.tus.server.AbstractITTusFileUploadService; +import me.desair.tus.server.ProtocolVersion; +import me.desair.tus.server.TestUtils; +import me.desair.tus.server.TusFileUploadService; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.testcontainers.containers.GenericContainer; + +/** + * End-to-end integration test suite verifying {@link TusFileUploadService} backed by {@link + * AzureBlobStorageService} and {@link AzureBlobLockingService} on Azurite using the official Azure + * Storage Blob SDK. Extends {@link AbstractITTusFileUploadService} to run all Tus 1.0.0 protocol + * use cases against Azure Blob Storage. + */ +public class ITAzureBlobTusFileUploadService extends AbstractITTusFileUploadService { + + private static GenericContainer azurite; + private static BlobContainerClient containerClient; + private static final String CONTAINER = "test-tus-azure-container"; + + @BeforeClass + public static void setUpClass() { + org.junit.Assume.assumeTrue( + "Container runtime is not available; skipping Testcontainers Azurite test", + TestUtils.isContainerRuntimeAvailable()); + + azurite = TestUtils.createAzuriteContainer(); + azurite.start(); + + containerClient = TestUtils.createBlobContainerClient(azurite, CONTAINER); + } + + @AfterClass + public static void tearDownClass() { + if (azurite != null) { + azurite.stop(); + } + } + + @Override + protected TusFileUploadService createTusFileUploadService() { + return createTusFileUploadService(UPLOAD_URI); + } + + @Override + protected TusFileUploadService createTusFileUploadService(String uploadUri) { + org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); + + AzureBlobStorageService azureStorage = new AzureBlobStorageService(containerClient); + AzureBlobLockingService azureLocking = new AzureBlobLockingService(containerClient); + AzureBlobConcatenationService azureConcat = + new AzureBlobConcatenationService(containerClient, azureStorage); + azureStorage.setUploadConcatenationService(azureConcat); + + return new TusFileUploadService() + .withUploadUri(uploadUri) + .withUploadStorageService(azureStorage) + .withUploadLockingService(azureLocking) + .withMaxUploadSize(1073741824L) + .withUploadExpirationPeriod(2L * 24 * 60 * 60 * 1000) + .withSupportedProtocolVersions(ProtocolVersion.TUS_1_0_0) + .withDownloadFeature() + .withChunkedTransferDecoding(true); + } +} diff --git a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java index 49ce17b2..d308cb88 100644 --- a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java @@ -215,5 +215,8 @@ public void testDelegateMethods() throws Exception { service.requestLockRelease("/files/1"); verify(mockLocking, times(1)).requestLockRelease("/files/1"); + + service.close(); + verify(mockLocking, times(1)).close(); } } diff --git a/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java index 3dc9bac8..b74ea409 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java @@ -654,4 +654,12 @@ public void testStopPathCreationExceptions() throws Exception { tempDir.toFile().setWritable(true); FileUtils.deleteDirectory(tempDir.toFile()); } + + @Test + public void testClose() throws Exception { + DiskLockingService service = new DiskLockingService(storagePath.toString()); + service.close(); + // Subsequent close should be idempotent no-op + service.close(); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java index e87fce8e..41a480bc 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java @@ -257,6 +257,16 @@ public void testGetConcatenatedBytesNull() throws Exception { assertNull(concatenationService.getConcatenatedBytes(null)); } + @Test + public void testGetConcatenatedBytesInProgressReturnsEmptyStream() throws Exception { + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("concat-in-progress")); + + InputStream is = concatenationService.getConcatenatedBytes(info); + assertNotNull(is); + assertEquals(0, is.available()); + } + /** * Tests that merging fails and throws an UploadNotFoundException when the child upload has a * different owner key than the parent/final upload in S3 storage. diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index 2f0c1e4b..aed5d1e1 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -269,4 +269,9 @@ public void testSanitizePrefixNullOrEmpty() throws Exception { new S3LockingService(minioClient, "test-bucket", null, 30000L, 0L); assertNotNull(serviceWithNullPrefix); } + + @Test + public void testClose() throws Exception { + lockingService.close(); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index f0fa45e0..2de3d8df 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -255,6 +255,17 @@ public void testGetUploadInfoThrowsIOExceptionOnGenericException() throws Except storageService.getUploadInfo(new UploadId("24249a5b-01a4-4bf8-b67a-364273bb5a2e")); } + @Test + public void testGetS3ObjectKeyWithDuplicatesUploadId() { + UploadInfo info = new UploadInfo(); + UploadId childId = new UploadId("child-id"); + UploadId parentId = new UploadId("parent-id"); + info.setId(childId); + info.setDuplicatesUploadId(parentId); + + assertEquals("uploads/parent-id", storageService.getS3ObjectKey(info)); + } + @Test(expected = IOException.class) public void testGetUploadInfoThrowsIOExceptionOnErrorResponseNon404() throws Exception { ErrorResponse errorResponse = mock(ErrorResponse.class); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java index 19262143..53ee607d 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java @@ -20,6 +20,9 @@ public void testParseErrorResponseCodes() throws Exception { assertEquals( S3ErrorType.NO_SUCH_KEY, S3Utils.parseErrorResponse(createExceptionWithCode("NoSuchBucket"))); + assertEquals( + S3ErrorType.NO_SUCH_KEY, + S3Utils.parseErrorResponse(createExceptionWithCode("NoSuchUpload"))); assertEquals( S3ErrorType.PRECONDITION_FAILED, S3Utils.parseErrorResponse(createExceptionWithCode("PreconditionFailed"))); @@ -29,6 +32,9 @@ public void testParseErrorResponseCodes() throws Exception { assertEquals( S3ErrorType.ACCESS_DENIED, S3Utils.parseErrorResponse(createExceptionWithCode("AccessDenied"))); + assertEquals( + S3ErrorType.API_NOT_IMPLEMENTED, + S3Utils.parseErrorResponse(createExceptionWithCode("APINotImplemented"))); assertEquals( S3ErrorType.UNKNOWN, S3Utils.parseErrorResponse(createExceptionWithCode("InternalError"))); } From c483d91b9c2c206947f314db77b8c686e0e06b94 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Tue, 11 Aug 2026 19:12:28 +0200 Subject: [PATCH 02/14] test(azure): manage testcontainer lifecycle in BeforeClass/AfterClass to skip gracefully without docker --- .../AzureBlobConcatenationServiceTest.java | 22 ++++++++++++++++--- .../azure/AzureBlobLockingServiceTest.java | 22 ++++++++++++++++--- .../azure/AzureBlobStorageServiceTest.java | 22 ++++++++++++++++--- .../upload/azure/AzureBlobUploadLockTest.java | 22 ++++++++++++++++--- 4 files changed, 76 insertions(+), 12 deletions(-) diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java index e0c76398..bed0323b 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java @@ -13,16 +13,32 @@ import me.desair.tus.server.exception.UploadNotFoundException; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadType; +import org.junit.AfterClass; import org.junit.Assume; import org.junit.Before; -import org.junit.ClassRule; +import org.junit.BeforeClass; import org.junit.Test; import org.testcontainers.containers.GenericContainer; public class AzureBlobConcatenationServiceTest { - @ClassRule - public static GenericContainer azuriteContainer = TestUtils.createAzuriteContainer(); + private static GenericContainer azuriteContainer; + + @BeforeClass + public static void setUpClass() { + Assume.assumeTrue( + "Container runtime is not available; skipping Testcontainers Azurite test", + TestUtils.isContainerRuntimeAvailable()); + azuriteContainer = TestUtils.createAzuriteContainer(); + azuriteContainer.start(); + } + + @AfterClass + public static void tearDownClass() { + if (azuriteContainer != null) { + azuriteContainer.stop(); + } + } private BlobContainerClient containerClient; private AzureBlobStorageService storageService; diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java index 29fbe917..c1357626 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java @@ -14,16 +14,32 @@ import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadLock; import me.desair.tus.server.util.InterruptibleInputStream; +import org.junit.AfterClass; import org.junit.Assume; import org.junit.Before; -import org.junit.ClassRule; +import org.junit.BeforeClass; import org.junit.Test; import org.testcontainers.containers.GenericContainer; public class AzureBlobLockingServiceTest { - @ClassRule - public static GenericContainer azuriteContainer = TestUtils.createAzuriteContainer(); + private static GenericContainer azuriteContainer; + + @BeforeClass + public static void setUpClass() { + Assume.assumeTrue( + "Container runtime is not available; skipping Testcontainers Azurite test", + TestUtils.isContainerRuntimeAvailable()); + azuriteContainer = TestUtils.createAzuriteContainer(); + azuriteContainer.start(); + } + + @AfterClass + public static void tearDownClass() { + if (azuriteContainer != null) { + azuriteContainer.stop(); + } + } private BlobContainerClient containerClient; private AzureBlobLockingService lockingService; diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java index d265021f..52583ac1 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java @@ -12,16 +12,32 @@ import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; +import org.junit.AfterClass; import org.junit.Assume; import org.junit.Before; -import org.junit.ClassRule; +import org.junit.BeforeClass; import org.junit.Test; import org.testcontainers.containers.GenericContainer; public class AzureBlobStorageServiceTest { - @ClassRule - public static GenericContainer azuriteContainer = TestUtils.createAzuriteContainer(); + private static GenericContainer azuriteContainer; + + @BeforeClass + public static void setUpClass() { + Assume.assumeTrue( + "Container runtime is not available; skipping Testcontainers Azurite test", + TestUtils.isContainerRuntimeAvailable()); + azuriteContainer = TestUtils.createAzuriteContainer(); + azuriteContainer.start(); + } + + @AfterClass + public static void tearDownClass() { + if (azuriteContainer != null) { + azuriteContainer.stop(); + } + } private BlobContainerClient containerClient; private AzureBlobStorageService storageService; diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java index c497e416..869074ff 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java @@ -8,16 +8,32 @@ import com.azure.storage.blob.specialized.BlobLeaseClientBuilder; import me.desair.tus.server.TestUtils; import me.desair.tus.server.upload.UploadLock; +import org.junit.AfterClass; import org.junit.Assume; import org.junit.Before; -import org.junit.ClassRule; +import org.junit.BeforeClass; import org.junit.Test; import org.testcontainers.containers.GenericContainer; public class AzureBlobUploadLockTest { - @ClassRule - public static GenericContainer azuriteContainer = TestUtils.createAzuriteContainer(); + private static GenericContainer azuriteContainer; + + @BeforeClass + public static void setUpClass() { + Assume.assumeTrue( + "Container runtime is not available; skipping Testcontainers Azurite test", + TestUtils.isContainerRuntimeAvailable()); + azuriteContainer = TestUtils.createAzuriteContainer(); + azuriteContainer.start(); + } + + @AfterClass + public static void tearDownClass() { + if (azuriteContainer != null) { + azuriteContainer.stop(); + } + } private BlobContainerClient containerClient; private String uploadUri; From 2211a064d3708a1b13d8738c3dc817ca2cf17822 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Tue, 11 Aug 2026 22:19:19 +0200 Subject: [PATCH 03/14] fix: vulnerabilities and code review --- .gitignore | 2 + docs/LOCKING.md | 100 +++++++++++------- pom.xml | 30 ++++++ .../upload/azure/AzureBlobUploadLock.java | 28 ++--- .../server/upload/s3/S3LockingService.java | 24 ++--- .../tus/server/upload/s3/S3UploadLock.java | 24 ++--- .../java/me/desair/tus/server/util/Utils.java | 54 ++++++++++ .../java/me/desair/tus/server/TestUtils.java | 2 +- .../me/desair/tus/server/util/UtilsTest.java | 60 +++++++++++ 9 files changed, 232 insertions(+), 92 deletions(-) diff --git a/.gitignore b/.gitignore index 814c5352..367dc812 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,5 @@ CONFORMITY_TEST_IMPROVEMENTS.md S3_STORAGE_ANALYSIS.md AZURE_BLOB_STORAGE_ANALYSIS.md AZURE_BLOB_STORAGE_IMPROVEMENTS.md +SFTP_STORAGE_ANALYSIS.md +NFS_LOCKING.md diff --git a/docs/LOCKING.md b/docs/LOCKING.md index c0de55f6..318507d4 100644 --- a/docs/LOCKING.md +++ b/docs/LOCKING.md @@ -1,78 +1,100 @@ # Upload Locking & Lock Contention Resolution -This document describes how the `tus-java-server` library prevents concurrent modifications to uploads using locks, and how it resolves lock contention when clients resume interrupted uploads. +This document describes why locking is necessary in the `tus-java-server` library, how the core `UploadLockingService` interface is structured, how lock contention resolution works across replicas, and where to find detailed documentation for each concrete locking mechanism implementation. --- ## 1. Why Locking is Needed -In the `tus` protocol, client uploads can be resumed after network interruptions. Multiple concurrent requests targeting the same upload resource must be prevented to avoid data corruption (e.g. out-of-order writes or overlapping file offsets). +In the `tus` protocol (and IETF Resumable Uploads for HTTP specification), client uploads can be interrupted and resumed across multiple HTTP requests. Multiple concurrent requests targeting the same upload resource must be strictly prevented to avoid data corruption (such as out-of-order byte writes or overlapping file offsets). -### Stalled uploads and resume handling -1. When a client performs an upload via a `PATCH` request, the server acquires an exclusive lock on that upload. -2. If the client's network connection drops, the `PATCH` request connection might remain in a "half-open" state on the server (stalled socket read). -3. The client, recognizing the disconnect, attempts to resume by sending a `HEAD` request to query the current offset (or a `DELETE` request to terminate/clean up the upload). -4. However, the stalled `PATCH` request is still running on the server and holding the lock, preventing the client from resuming or deleting. +### Stalled Uploads & Lock Contention Handling -To solve this, we need a mechanism where a new `HEAD` or `DELETE` request can trigger the release of the lock held by the stalled request, allowing immediate resumability or deletion. +1. **Active Streaming**: When a client sends upload bytes via a `PATCH` (or RUFH `POST`/`PATCH`) request, the server acquires an exclusive lock on that upload. +2. **Network Interruption**: If the client's network drops, the original `PATCH` connection may remain open on the server in a "half-open" state (a stalled socket read waiting for client bytes). +3. **Resume Attempt**: The client, recognizing the disconnect, attempts to resume by sending a `HEAD` request to query the current offset (or a `DELETE` request to terminate the upload). +4. **Lock Conflict**: The stalled `PATCH` request is still running on the server and holding the lock, which would block the client's `HEAD` or `DELETE` request indefinitely if not resolved. + +To solve this, `tus-java-server` includes a **lock contention resolution mechanism** where an incoming `HEAD` or `DELETE` request signals the server to interrupt the stalled `PATCH` byte stream cleanly, releasing the lock for immediate resumption. --- -## 2. High-Level Interface (`UploadLockingService`) +## 2. Core Locking Interfaces + +All locking mechanisms in `tus-java-server` implement the `UploadLockingService` interface and return handles implementing `UploadLock`. -The locking behaviour is defined by the `UploadLockingService` interface. To support backwards compatibility and lock contention resolution, the interface exposes the following high-level API: +### `UploadLockingService` Interface ```java -public interface UploadLockingService { +public interface UploadLockingService extends Closeable { - // Acquires a lock on an upload resource + // Acquires an exclusive lock on an upload resource UploadLock lockUploadByUri(String requestUri) throws TusException, IOException; // Checks if an upload is currently locked boolean isLocked(UploadId id); - // Cleans up stale locks left on disk + // Cleans up stale or expired locks void cleanupStaleLocks() throws IOException; - // Registers the input stream of an active request so it can be interrupted later + // Registers the active request input stream so it can be interrupted cleanly default void registerInputStream(String requestUri, InputStream inputStream) {} // Requests that any active lock for the URI be released default void requestLockRelease(String requestUri) {} + + // Injects the UploadIdFactory instance used to parse upload IDs from request URIs + default void setIdFactory(UploadIdFactory idFactory) {} + + // Injects the upload expiration period in milliseconds + default void setUploadExpirationPeriod(Long expirationPeriod) {} } ``` -- **Backward Compatibility**: Both `registerInputStream` and `requestLockRelease` are `default` (no-op) methods, ensuring that third-party custom implementations of `UploadLockingService` (e.g. S3, Redis, or Database backends) do not break. -- **Request Flow**: - - When a `PATCH` request stream is created, its input stream is wrapped in an `InterruptibleInputStream` and registered via `registerInputStream`. - - When a `HEAD` or `DELETE` request encounters a lock conflict, it invokes `requestLockRelease`, which triggers the watchdog and/or local interruption. +### `UploadLock` Interface + +```java +public interface UploadLock extends Closeable { + + // Gets the request URI associated with this lock + String getUploadUri(); + + // Explicitly releases the lock + default void release() { + try { + close(); + } catch (IOException ignored) {} + } +} +``` + +### Request Flow & Contention Resolution + +- **Stream Registration**: When a request starts streaming payload bytes, its input stream is wrapped in an `InterruptibleInputStream` and registered via `lockingService.registerInputStream(requestUri, inputStream)`. +- **Release Request**: When a concurrent `HEAD` or `DELETE` request encounters an active lock, `TusFileUploadService` catches `UploadAlreadyLockedException` and calls `lockingService.requestLockRelease(requestUri)`. +- **Stream Interruption**: + - If the lock is held in the **same JVM**, `requestLockRelease` interrupts the local stream directly. + - If the lock is held on a **remote replica/pod**, `requestLockRelease` writes a `.stop` signal object or file. A background watchdog thread running on the lock-holding replica detects the `.stop` signal and calls `stream.interrupt()`, causing the stalled `PATCH` stream to abort and release its lock. --- -## 3. File System Based Implementation (`DiskLockingService`) +## 3. Concrete Locking Mechanisms & Storage Providers -### 3.1. General Overview +`tus-java-server` provides several built-in locking implementations tailored for different deployment topologies and storage backends. Refer to the dedicated documentation files below for full details: -`DiskLockingService` is the default locking service. It implements locking using Java NIO `FileChannel` and exclusive `FileLock` objects. +| Storage Backend / Environment | Locking Service Class | Key Characteristics & Architecture | Documentation File | +|---|---|---|---| +| **Local File System** | `DiskLockingService` | OS kernel-level exclusive POSIX `FileLock` (`fcntl`) with JVM shutdown hooks and `.stop` signal files. Best for single-node deployments on local disk. | [`README.md`](file:///Users/tom/projects/tus-java-server/README.md) | +| **Amazon S3 / S3-Compatible** | `S3LockingService` | MinIO/S3 object-backed TTL lease objects (`.lock`), background heartbeat renewal, and cross-pod `.stop` signal object polling watchdog. | [`docs/S3_STORAGE.md`](file:///Users/tom/projects/tus-java-server/docs/S3_STORAGE.md) | +| **Azure Blob Storage** | `AzureBlobLockingService` | Native Azure Blob Storage exclusive 30-second leases (`BlobLeaseClient`), background daemon renewal, and `.stop` signal blob polling watchdog. | [`docs/AZURE_BLOB_STORAGE.md`](file:///Users/tom/projects/tus-java-server/docs/AZURE_BLOB_STORAGE.md) | -- **Lock Files**: For an upload ID ``, it attempts to acquire an exclusive lock on the file `locks/` in the storage directory. -- **Cross-Replica / Multi-Process Signaling**: - - In a clustered or multi-container setup (e.g. Kubernetes with a shared Persistent Volume Claim (PVC)), different server instances may handle different requests. - - When `requestLockRelease` is called, it: - 1. Interrupts the local stream if the lock is held in the same JVM. - 2. Writes an empty `.stop` file named `locks/.stop` in the shared locks directory. - - The JVM instance that currently holds the file lock detects this `.stop` file and terminates its request. +--- -### 3.2. The Watchdog Process +## 4. Implementing a Custom `UploadLockingService` -The watchdog is a background daemon thread managed entirely inside `DiskLockingService`. +Developers extending `tus-java-server` with custom lock providers (such as Redis, ZooKeeper, etcd, or Hazelcast) must implement `UploadLockingService` and `UploadLock`: -### Role & Lifecycle -- **Triggered**: Spawns automatically when a request registers its stream in the JVM-local `activeLocks` registry. -- **Polling Loop**: Every 1 second, it scans all active locks. If a `.stop` file exists for a given upload ID, it invokes `stream.interrupt()`, which immediately terminates the stalled connection. -- **Self-Termination**: To conserve resources, the watchdog thread terminates naturally when there are no more active locks in the registry. It will spawn a new thread if a new upload request starts. -- **Safety**: - - Runs with `Thread.MIN_PRIORITY` to avoid stealing CPU cycles from request handling threads. - - Set as a daemon thread (`setDaemon(true)`) so it does not block application/JVM shutdown. - - Uses `WeakReference` for active locks to prevent memory leaks if a request thread terminates unexpectedly without cleaning up its lock. - - Catches `Throwable` inside the loop to ensure unexpected errors do not crash the daemon silently. +1. **Implement `lockUploadByUri`**: Acquire an exclusive lock or throw `UploadAlreadyLockedException` if currently locked by another request. +2. **Implement `registerInputStream` & `requestLockRelease`**: Maintain a registry of active `InterruptibleInputStream` instances. When `requestLockRelease` is invoked, interrupt the active stream to support instant client resumes. +3. **Use Common Watchdog Helpers**: Use `Utils.scheduleWatchdog(...)` and `Utils.shutdownExecutor(...)` in `me.desair.tus.server.util.Utils` for any background daemon threads or heartbeat lease renewals. +4. **Implement `Closeable`**: Register a JVM shutdown hook upon construction and deregister it on `close()` for idempotent resource cleanup. diff --git a/pom.xml b/pom.xml index 26936d61..2831a2a0 100644 --- a/pom.xml +++ b/pom.xml @@ -67,6 +67,36 @@ 12.35.0 provided + + io.netty + netty-codec + 4.2.17.Final + provided + + + io.netty + netty-codec + 4.2.17.Final + provided + + + io.netty + netty-codec-http + 4.2.17.Final + provided + + + io.netty + netty-codec-http2 + 4.2.17.Final + provided + + + io.netty + netty-codec-dns + 4.2.17.Final + provided + diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java index 76970e98..af3f6a06 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java @@ -4,10 +4,10 @@ import com.azure.storage.blob.specialized.BlobLeaseClient; import java.io.IOException; import java.util.Objects; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import me.desair.tus.server.upload.UploadLock; +import me.desair.tus.server.util.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,20 +45,12 @@ public AzureBlobUploadLock(BlobLeaseClient leaseClient, BlobClient lockBlob, Str // Initialize background daemon thread to renew lease periodically during upload this.renewalExecutor = - Executors.newSingleThreadScheduledExecutor( - runnable -> { - Thread thread = new Thread(runnable, "azure-lease-renewal-" + uploadUri); - thread.setDaemon(true); - return thread; - }); - - scheduleLeaseRenewal(); - } - - /** Schedules periodic background renewal of the active lease. */ - private void scheduleLeaseRenewal() { - renewalExecutor.scheduleAtFixedRate( - this::renewLease, RENEWAL_INTERVAL_SECONDS, RENEWAL_INTERVAL_SECONDS, TimeUnit.SECONDS); + Utils.scheduleWatchdog( + "azure-lease-renewal-" + uploadUri, + this::renewLease, + RENEWAL_INTERVAL_SECONDS, + RENEWAL_INTERVAL_SECONDS, + TimeUnit.SECONDS); } /** Attempts to renew the lease with Azure Blob Storage. */ @@ -106,10 +98,6 @@ public String getUploadUri() { /** Shuts down the renewal executor cleanly. */ private void shutdownExecutor() { - try { - renewalExecutor.shutdownNow(); - } catch (Exception ignored) { - // Ignore shutdown interrupts - } + Utils.shutdownExecutor(renewalExecutor); } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index e7d857a4..620aa58e 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -17,7 +17,6 @@ import java.util.Objects; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import me.desair.tus.server.exception.TusException; @@ -29,6 +28,7 @@ import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.util.InterruptibleInputStream; import me.desair.tus.server.util.S3UploadLockJsonSerializer; +import me.desair.tus.server.util.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,17 +115,12 @@ public S3LockingService( // Background watchdog thread to poll S3 for .stop contention signals across pods this.watchdogExecutor = - Executors.newSingleThreadScheduledExecutor( - r -> { - Thread t = new Thread(r, "s3-lock-watchdog"); - t.setDaemon(true); - return t; - }); - - if (pollIntervalMs > 0) { - this.watchdogExecutor.scheduleAtFixedRate( - this::checkStopSignals, pollIntervalMs, pollIntervalMs, TimeUnit.MILLISECONDS); - } + Utils.scheduleWatchdog( + "s3-lock-watchdog", + this::checkStopSignals, + pollIntervalMs, + pollIntervalMs, + TimeUnit.MILLISECONDS); // Register automatic JVM shutdown hook to clean up watchdog thread pool on app/pod shutdown this.shutdownHook = new Thread(this::closeQuietly, "s3-lock-shutdown-hook"); @@ -322,10 +317,7 @@ public void close() throws IOException { if (!closed) { closed = true; deregisterShutdownHook(); - try { - watchdogExecutor.shutdownNow(); - } catch (Exception ignored) { - } + Utils.shutdownExecutor(watchdogExecutor); activeInputStreams.clear(); } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java index 4a4303d7..91d43a79 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java @@ -9,11 +9,11 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.Map; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import me.desair.tus.server.upload.UploadLock; import me.desair.tus.server.util.S3UploadLockJsonSerializer; +import me.desair.tus.server.util.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -129,14 +129,12 @@ public S3UploadLock( // lease) long heartbeatPeriodMs = Math.max(1000L, leaseDurationMs / 3); this.heartbeatExecutor = - Executors.newSingleThreadScheduledExecutor( - r -> { - Thread t = new Thread(r, "s3-lock-heartbeat-" + holderId); - t.setDaemon(true); - return t; - }); - this.heartbeatExecutor.scheduleAtFixedRate( - this::renewLease, heartbeatPeriodMs, heartbeatPeriodMs, TimeUnit.MILLISECONDS); + Utils.scheduleWatchdog( + "s3-lock-heartbeat-" + holderId, + this::renewLease, + heartbeatPeriodMs, + heartbeatPeriodMs, + TimeUnit.MILLISECONDS); } S3UploadLock( @@ -239,13 +237,7 @@ public void release() { @Override public void close() { // Step 1: Stop the background heartbeat daemon thread - if (heartbeatExecutor != null) { - try { - heartbeatExecutor.shutdownNow(); - } catch (Exception e) { - log.debug("Error shutting down lock heartbeat executor", e); - } - } + Utils.shutdownExecutor(heartbeatExecutor); // Step 2: Remove active request stream registration if (inputStreamMap != null && requestUri != null) { diff --git a/src/main/java/me/desair/tus/server/util/Utils.java b/src/main/java/me/desair/tus/server/util/Utils.java index 94da342e..b93a7b11 100644 --- a/src/main/java/me/desair/tus/server/util/Utils.java +++ b/src/main/java/me/desair/tus/server/util/Utils.java @@ -19,6 +19,9 @@ import java.util.EnumSet; import java.util.LinkedList; import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; @@ -389,4 +392,55 @@ public static String getUploadUriOnCreation( uploadInfo != null && uploadInfo.getId() != null ? uploadInfo.getId().toString() : ""; return baseUri + (baseUri.endsWith("/") ? "" : "/") + idStr; } + + /** + * Creates a single-thread scheduled executor service with a daemon thread of the given name. + * + * @param threadName The name for the background daemon thread + * @return A new single-thread ScheduledExecutorService + */ + public static ScheduledExecutorService createScheduledDaemonExecutor(String threadName) { + return Executors.newSingleThreadScheduledExecutor( + runnable -> { + Thread thread = new Thread(runnable, threadName); + thread.setDaemon(true); + return thread; + }); + } + + /** + * Creates a daemon scheduled executor and immediately schedules a task to run periodically at a + * fixed rate. + * + * @param threadName The name for the background daemon thread + * @param task The task to execute periodically + * @param initialDelay The initial delay before the first execution + * @param period The period between successive executions + * @param unit The time unit of the initialDelay and period parameters + * @return The created ScheduledExecutorService + */ + public static ScheduledExecutorService scheduleWatchdog( + String threadName, Runnable task, long initialDelay, long period, TimeUnit unit) { + ScheduledExecutorService executor = createScheduledDaemonExecutor(threadName); + if (period > 0 && task != null) { + executor.scheduleAtFixedRate(task, initialDelay, period, unit); + } + return executor; + } + + /** + * Safely shuts down a ScheduledExecutorService using {@link + * ScheduledExecutorService#shutdownNow()}. + * + * @param executor The ScheduledExecutorService to shut down + */ + public static void shutdownExecutor(ScheduledExecutorService executor) { + if (executor != null && !executor.isShutdown()) { + try { + executor.shutdownNow(); + } catch (Exception e) { + log.debug("Error shutting down executor: {}", e.getMessage()); + } + } + } } diff --git a/src/test/java/me/desair/tus/server/TestUtils.java b/src/test/java/me/desair/tus/server/TestUtils.java index db0b1bf1..3d698f76 100644 --- a/src/test/java/me/desair/tus/server/TestUtils.java +++ b/src/test/java/me/desair/tus/server/TestUtils.java @@ -111,7 +111,7 @@ public static void createBucket(MinioClient minioClient, String bucket) { * @return A configured GenericContainer instance (not started yet) */ public static GenericContainer createAzuriteContainer() { - return new GenericContainer<>("mcr.microsoft.com/azure-storage/azurite:latest") + return new GenericContainer<>("mcr.microsoft.com/azure-storage/azurite:3.36.0") .withExposedPorts(10000) .withCommand("azurite-blob", "--blobHost", "0.0.0.0", "--skipApiVersionCheck"); } diff --git a/src/test/java/me/desair/tus/server/util/UtilsTest.java b/src/test/java/me/desair/tus/server/util/UtilsTest.java index bbd4c681..6f7f6131 100644 --- a/src/test/java/me/desair/tus/server/util/UtilsTest.java +++ b/src/test/java/me/desair/tus/server/util/UtilsTest.java @@ -512,6 +512,66 @@ public void testIsExistingUploadResourceNullRequestUri() throws Exception { assertThat(Utils.isExistingUploadResource(request, storageService, "owner"), is(false)); } + @Test + public void testCreateScheduledDaemonExecutorAndScheduleWatchdog() throws Exception { + java.util.concurrent.atomic.AtomicBoolean executed = + new java.util.concurrent.atomic.AtomicBoolean(false); + java.util.concurrent.ScheduledExecutorService executor = + Utils.scheduleWatchdog( + "test-watchdog", + () -> executed.set(true), + 10, + 10, + java.util.concurrent.TimeUnit.MILLISECONDS); + + assertThat(executor, is(notNullValue())); + assertThat(executor.isShutdown(), is(false)); + + Thread.sleep(50); + assertThat(executed.get(), is(true)); + + Utils.shutdownExecutor(executor); + assertThat(executor.isShutdown(), is(true)); + } + + @Test + public void testShutdownExecutorNullOrShutdown() { + Utils.shutdownExecutor(null); + + java.util.concurrent.ScheduledExecutorService executor = + Utils.createScheduledDaemonExecutor("test-shutdown"); + Utils.shutdownExecutor(executor); + assertThat(executor.isShutdown(), is(true)); + + Utils.shutdownExecutor(executor); + assertThat(executor.isShutdown(), is(true)); + } + + @Test + public void testScheduleWatchdogWithZeroPeriodOrNullTask() { + java.util.concurrent.ScheduledExecutorService executor1 = + Utils.scheduleWatchdog( + "test-zero-period", () -> {}, 0, 0, java.util.concurrent.TimeUnit.SECONDS); + assertThat(executor1, is(notNullValue())); + Utils.shutdownExecutor(executor1); + + java.util.concurrent.ScheduledExecutorService executor2 = + Utils.scheduleWatchdog( + "test-null-task", null, 10, 10, java.util.concurrent.TimeUnit.SECONDS); + assertThat(executor2, is(notNullValue())); + Utils.shutdownExecutor(executor2); + } + + @Test + public void testShutdownExecutorWithException() { + java.util.concurrent.ScheduledExecutorService mockExecutor = + mock(java.util.concurrent.ScheduledExecutorService.class); + when(mockExecutor.isShutdown()).thenReturn(false); + when(mockExecutor.shutdownNow()).thenThrow(new RuntimeException("Shutdown error")); + + Utils.shutdownExecutor(mockExecutor); + } + /** Simple serializable class for testing. */ public static class TestSerializable implements Serializable { private static final long serialVersionUID = 1L; From 0154a835f11e664cf49afed1897963a81d74be46 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Tue, 11 Aug 2026 22:23:18 +0200 Subject: [PATCH 04/14] fix: Add enable-final-field-mutation=ALL-UNNAMED to remove warnings --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2831a2a0..6accc62d 100644 --- a/pom.xml +++ b/pom.xml @@ -277,7 +277,7 @@ 3.2.5 - ${surefireArgLine} + ${surefireArgLine} --enable-final-field-mutation=ALL-UNNAMED ${project.build.directory}/surefire-reports @@ -296,7 +296,7 @@ 3.2.5 - ${failsafeArgLine} + ${failsafeArgLine} --enable-final-field-mutation=ALL-UNNAMED ${project.build.directory}/surefire-reports From afc85909987a8d35d6f2f00c1c4bf9fbc6de012a Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Tue, 11 Aug 2026 23:02:51 +0200 Subject: [PATCH 05/14] feat: increase test coverage --- AGENTS.md | 12 + pom.xml | 4 +- scripts/check-coverage.py | 110 +++- .../upload/azure/AzureBlobLockingService.java | 2 +- .../upload/azure/AzureBlobUploadLock.java | 2 +- .../azure/AzureBlobLockingServiceTest.java | 114 ---- .../azure/AzureBlobStorageServiceTest.java | 195 ------- .../upload/azure/AzureBlobUploadLockTest.java | 78 --- ...a => ITAzureBlobConcatenationService.java} | 103 +++- .../azure/ITAzureBlobLockingService.java | 210 ++++++++ .../azure/ITAzureBlobStorageService.java | 496 ++++++++++++++++++ .../azure/ITAzureBlobStorageServiceTest.java | 128 ----- .../upload/azure/ITAzureBlobUploadLock.java | 132 +++++ 13 files changed, 1039 insertions(+), 547 deletions(-) delete mode 100644 src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java delete mode 100644 src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java delete mode 100644 src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java rename src/test/java/me/desair/tus/server/upload/azure/{AzureBlobConcatenationServiceTest.java => ITAzureBlobConcatenationService.java} (57%) create mode 100644 src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java create mode 100644 src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java delete mode 100644 src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageServiceTest.java create mode 100644 src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobUploadLock.java diff --git a/AGENTS.md b/AGENTS.md index 64e8ba20..457f1ace 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -132,6 +132,18 @@ To maximize developer velocity and minimize test execution overhead when increas - **Fast Unit Test Execution**: Verify all local unit tests rapidly using target wildcard patterns (e.g. `mvn test -Dtest="S3*" -q` or `mvn test -Dtest="*Test" -q`). Unit tests run in under 2 seconds without launching test containers. - **Single Verification Gate**: Only run the full JaCoCo diff coverage verification command (`mvn verify -Pcheck-coverage -Djacoco.compare.branch=master -q`) after all batched unit test updates have been applied and locally validated. +### 19. Consolidated UT + IT Code Coverage Verification & Preventing GitHub CI Failures +To ensure code coverage checks never fail in GitHub Actions CI or PR validation pipelines: +- **Consolidated Coverage Script (`scripts/check-coverage.py`)**: + - The coverage verification script automatically discovers and aggregates coverage across **both** unit tests (`target/site/jacoco-ut/jacoco.xml`) and integration tests (`target/site/jacoco-it/jacoco.xml`). + - Supports `--filter` (e.g., `--filter azure`), `--per-file-limit` (e.g., `--per-file-limit 90`), `--limit` (overall threshold), and `--compare-branch` (checking diff coverage on modified lines against a base git branch). + - Outlines exact uncovered and partially covered line number ranges (e.g., `103, 112, 160-165`) for fast diagnostic and test creation. +- **Local Verification Gate**: Before committing or pushing changes to GitHub, always execute: + ```bash + mvn verify -Pcheck-coverage -Djacoco.compare.branch=master -q + ``` +- **Integration Test Class Naming & Handling**: Integration test classes (classes that rely on containers or Testcontainers) MUST start with `IT` and MUST NOT end with `Test` or `Test.java` (e.g., `ITAzureBlobStorageService.java`, `ITAzureBlobConcatenationService.java`). This ensures Maven Surefire skips them during `mvn test` and Maven Failsafe runs them during `mvn verify`. When a container runtime is unavailable, integration test classes must be cleanly skipped via `Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable())`. Do not duplicate unit tests or introduce unnecessary mocking inside integration test classes. + ## IETF Resumable Uploads for HTTP (RUFH) Spec Maintenance & Update Playbook ### 1. Spec Diff Review diff --git a/pom.xml b/pom.xml index 6accc62d..6b1897c9 100644 --- a/pom.xml +++ b/pom.xml @@ -277,7 +277,7 @@ 3.2.5 - ${surefireArgLine} --enable-final-field-mutation=ALL-UNNAMED + ${surefireArgLine} --enable-final-field-mutation=ALL-UNNAMED --enable-native-access=ALL-UNNAMED ${project.build.directory}/surefire-reports @@ -296,7 +296,7 @@ 3.2.5 - ${failsafeArgLine} --enable-final-field-mutation=ALL-UNNAMED + ${failsafeArgLine} --enable-final-field-mutation=ALL-UNNAMED --enable-native-access=ALL-UNNAMED ${project.build.directory}/surefire-reports diff --git a/scripts/check-coverage.py b/scripts/check-coverage.py index 9e9ef242..0a9994f6 100644 --- a/scripts/check-coverage.py +++ b/scripts/check-coverage.py @@ -3,10 +3,15 @@ import argparse import subprocess +import os +import glob + def parse_args(): parser = argparse.ArgumentParser(description="Check Jacoco coverage limits and report uncovered lines.") - parser.add_argument("--xml", required=True, nargs="+", help="Path(s) to jacoco.xml files") - parser.add_argument("--limit", type=float, default=95.0, help="Minimum line coverage percentage (0-100)") + parser.add_argument("--xml", required=False, nargs="*", default=None, help="Path(s) or glob(s) to jacoco.xml files. If omitted, auto-discovers UT and IT reports.") + parser.add_argument("--limit", type=float, default=95.0, help="Minimum overall line coverage percentage (0-100)") + parser.add_argument("--per-file-limit", type=float, default=None, help="Minimum per-file line coverage percentage (0-100)") + parser.add_argument("--filter", type=str, default=None, help="Filter file paths by substring (e.g., 'azure')") parser.add_argument("--compare-branch", help="Compare against a git branch and check coverage of new/modified lines only") return parser.parse_args() @@ -78,17 +83,48 @@ def get_modified_lines(compare_branch): elif line.startswith("-") and not line.startswith("---"): pass else: - if line.startswith(" ") or line.startswith("\\"): - if line.startswith(" "): - current_line += 1 + if line.startswith(" "): + current_line += 1 return modified_lines +def resolve_xml_paths(xml_args): + if xml_args: + resolved = [] + for arg in xml_args: + matches = glob.glob(arg) + if matches: + resolved.extend(matches) + elif os.path.exists(arg): + resolved.append(arg) + return sorted(list(set(resolved))) + + # Auto-discovery default: collect all existing Jacoco report paths (UT, IT, merged, aggregate) + candidate_paths = [ + "target/site/jacoco-ut/jacoco.xml", + "target/site/jacoco-it/jacoco.xml", + "target/site/jacoco/jacoco.xml", + "target/site/jacoco-aggregate/jacoco.xml" + ] + return [p for p in candidate_paths if os.path.exists(p)] + def main(): args = parse_args() + xml_files = resolve_xml_paths(args.xml) + if not xml_files: + print("Error: No valid Jacoco XML reports could be found.") + print("Expected XML reports at 'target/site/jacoco-ut/jacoco.xml' or 'target/site/jacoco-it/jacoco.xml'.") + print("Please run 'mvn test' or 'mvn verify' first to generate coverage reports.") + sys.exit(1) + + print(f"📊 Consolidating Jacoco Coverage Reports from ({len(xml_files)} files):") + for f in xml_files: + print(f" • {f}") + print("----------------------------------------------------------") + parsed_trees = [] - for xml_path in args.xml: + for xml_path in xml_files: try: tree = ET.parse(xml_path) parsed_trees.append((xml_path, tree)) @@ -97,10 +133,9 @@ def main(): if not parsed_trees: print("Error: No valid Jacoco XML reports could be parsed.") - print("Please ensure you run 'mvn test' or 'mvn verify' first to generate coverage reports.") sys.exit(1) - # Aggregate coverage data across all reports: + # Aggregate coverage data across all reports (UT + IT): # coverage_data: full_path -> { line_nr -> { "mi": [], "ci": [], "mb": [], "cb": [] } } coverage_data = {} @@ -112,6 +147,9 @@ def main(): sf_name = sf.attrib.get("name", "") full_path = f"src/main/java/{pkg_name}/{sf_name}" + if args.filter and args.filter.lower() not in full_path.lower(): + continue + if full_path not in coverage_data: coverage_data[full_path] = {} @@ -147,7 +185,7 @@ def main(): total = total_covered + total_missed if total == 0: - print("No line coverage data found in report.") + print("No line coverage data found in reports for matching filter.") sys.exit(0) covered_pct = (total_covered / total) * 100.0 @@ -163,8 +201,10 @@ def main(): print(f"Failed to get modified lines against {compare_branch}. Aborting.") sys.exit(1) - # Find all uncovered files and lines + # Find all uncovered files and lines with per-file stats uncovered_files = [] + file_summaries = [] + failed_per_file_limit = False for full_path in sorted(coverage_data.keys()): # If we are filtering by modified lines, check if this file is modified @@ -173,6 +213,8 @@ def main(): missed_lines = [] partially_covered_lines = [] + file_covered = 0 + file_missed = 0 file_lines = coverage_data[full_path] for nr in sorted(file_lines.keys()): @@ -187,14 +229,36 @@ def main(): best_mb = min(line_info["mb"]) best_cb = max(line_info["cb"]) + if best_ci > 0: + file_covered += 1 + elif best_mi > 0: + file_missed += 1 + if best_mi > 0 and best_ci == 0: missed_lines.append(nr) elif (best_mi > 0 and best_ci > 0) or (best_mb > 0 and best_cb > 0): partially_covered_lines.append(nr) + file_total = file_covered + file_missed + file_pct = (file_covered / file_total * 100.0) if file_total > 0 else 100.0 + + if args.per_file_limit is not None and file_pct < args.per_file_limit: + failed_per_file_limit = True + + file_summaries.append({ + "file": full_path, + "pct": file_pct, + "covered": file_covered, + "missed_cnt": file_missed, + "total": file_total, + "missed_lines": missed_lines, + "partial_lines": partially_covered_lines + }) + if missed_lines or partially_covered_lines: uncovered_files.append({ "file": full_path, + "pct": file_pct, "missed": missed_lines, "partial": partially_covered_lines }) @@ -210,7 +274,7 @@ def main(): print("Uncovered / Partially Covered Modified Lines:") for uf in uncovered_files: file_path = uf["file"] - print(f"\n📄 {file_path}:") + print(f"\n📄 {file_path} ({uf['pct']:.2f}%):") if uf["missed"]: print(f" ❌ Uncovered lines: {group_ranges(uf['missed'])}") if uf["partial"]: @@ -230,7 +294,7 @@ def main(): print("🎉 Modified line coverage meets required threshold!") sys.exit(0) else: - print("🎉 All new and modified lines are 100% covered by unit tests!") + print("🎉 All new and modified lines are 100% covered by tests!") print("==========================================================") sys.exit(0) else: @@ -240,23 +304,23 @@ def main(): print(f"Overall Line Coverage: {covered_pct:.2f}% (Required: {args.limit:.2f}%)") print(f"Covered Lines: {total_covered}, Missed Lines: {total_missed}, Total Lines: {total}") print("----------------------------------------------------------") - if uncovered_files: - print("Uncovered / Partially Covered Files and Lines:") - for uf in uncovered_files: - file_path = uf["file"] - print(f"\n📄 {file_path}:") - if uf["missed"]: - print(f" ❌ Uncovered lines: {group_ranges(uf['missed'])}") - if uf["partial"]: - print(f" ⚠️ Partially covered lines: {group_ranges(uf['partial'])}") - else: - print("🎉 100% of all lines are fully covered by tests!") + print("Per-File Coverage Summary:") + for fs in file_summaries: + status_icon = "✅" if (args.per_file_limit is None or fs["pct"] >= args.per_file_limit) else "❌" + print(f"\n{status_icon} 📄 {fs['file']}: {fs['pct']:.2f}% ({fs['covered']}/{fs['total']} lines)") + if fs["missed_lines"]: + print(f" ❌ Uncovered lines: {group_ranges(fs['missed_lines'])}") + if fs["partial_lines"]: + print(f" ⚠️ Partially covered lines: {group_ranges(fs['partial_lines'])}") print("==========================================================") if covered_pct < args.limit: print(f"❌ FAIL: Line coverage is below threshold of {args.limit:.2f}%!") sys.exit(1) + elif failed_per_file_limit: + print(f"❌ FAIL: One or more files are below per-file limit of {args.per_file_limit:.2f}%!") + sys.exit(1) else: print("✅ SUCCESS: Coverage threshold check passed.") sys.exit(0) diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java index f6df3ab2..2a4f92a2 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java @@ -248,7 +248,7 @@ private void deleteStopSignalBlob(String idStr) { } /** Ensures the lock target blob exists on Azure Blob Storage. */ - private void ensureLockBlobExists(BlobClient lockBlob) { + void ensureLockBlobExists(BlobClient lockBlob) { try { if (!lockBlob.exists()) { lockBlob.upload(BinaryData.fromBytes("lock".getBytes(StandardCharsets.UTF_8)), false); diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java index af3f6a06..e5ae9bb2 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java @@ -54,7 +54,7 @@ public AzureBlobUploadLock(BlobLeaseClient leaseClient, BlobClient lockBlob, Str } /** Attempts to renew the lease with Azure Blob Storage. */ - private void renewLease() { + void renewLease() { if (released) { return; } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java deleted file mode 100644 index c1357626..00000000 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java +++ /dev/null @@ -1,114 +0,0 @@ -package me.desair.tus.server.upload.azure; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import com.azure.storage.blob.BlobContainerClient; -import java.io.ByteArrayInputStream; -import me.desair.tus.server.TestUtils; -import me.desair.tus.server.exception.UploadAlreadyLockedException; -import me.desair.tus.server.upload.TimeBasedUploadIdFactory; -import me.desair.tus.server.upload.UploadId; -import me.desair.tus.server.upload.UploadLock; -import me.desair.tus.server.util.InterruptibleInputStream; -import org.junit.AfterClass; -import org.junit.Assume; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.testcontainers.containers.GenericContainer; - -public class AzureBlobLockingServiceTest { - - private static GenericContainer azuriteContainer; - - @BeforeClass - public static void setUpClass() { - Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers Azurite test", - TestUtils.isContainerRuntimeAvailable()); - azuriteContainer = TestUtils.createAzuriteContainer(); - azuriteContainer.start(); - } - - @AfterClass - public static void tearDownClass() { - if (azuriteContainer != null) { - azuriteContainer.stop(); - } - } - - private BlobContainerClient containerClient; - private AzureBlobLockingService lockingService; - - @Before - public void setUp() { - Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - containerClient = - TestUtils.createBlobContainerClient( - azuriteContainer, "lock-unit-container-" + System.nanoTime()); - lockingService = new AzureBlobLockingService(containerClient); - TimeBasedUploadIdFactory idFactory = new TimeBasedUploadIdFactory(); - idFactory.setUploadUri("/test/upload"); - lockingService.setIdFactory(idFactory); - } - - @Test - public void lockUploadByUriShouldReturnNullOnInvalidUri() throws Exception { - assertNull(lockingService.lockUploadByUri("invalid-uri-no-id")); - } - - @Test - public void isLockedShouldReturnFalseWhenLockBlobDoesNotExist() { - assertFalse(lockingService.isLocked(new UploadId("12345"))); - } - - @Test - public void lockUploadByUriShouldAcquireLock() throws Exception { - UploadLock lock = lockingService.lockUploadByUri("/test/upload/12345"); - assertNotNull(lock); - assertEquals("/test/upload/12345", lock.getUploadUri()); - assertTrue(lockingService.isLocked(new UploadId("12345"))); - lock.release(); - } - - @Test(expected = UploadAlreadyLockedException.class) - public void lockUploadByUriShouldThrowOnLockContention() throws Exception { - UploadLock lock1 = lockingService.lockUploadByUri("/test/upload/12345"); - assertNotNull(lock1); - try { - // Second lock attempt on same URI should throw UploadAlreadyLockedException - lockingService.lockUploadByUri("/test/upload/12345"); - } finally { - lock1.release(); - } - } - - @Test - public void registerInputStreamAndRequestReleaseShouldInterruptStream() throws Exception { - ByteArrayInputStream bais = new ByteArrayInputStream("data".getBytes()); - InterruptibleInputStream stream = new InterruptibleInputStream(bais); - - lockingService.registerInputStream("/test/upload/12345", stream); - lockingService.requestLockRelease("/test/upload/12345"); - - try { - stream.read(); - } catch (Exception e) { - assertNotNull(e); - } - } - - @Test - public void cleanupStaleLocksShouldNotThrow() throws Exception { - lockingService.cleanupStaleLocks(); - } - - @Test - public void closeShouldCleanUpResources() throws Exception { - lockingService.close(); - } -} diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java deleted file mode 100644 index 52583ac1..00000000 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java +++ /dev/null @@ -1,195 +0,0 @@ -package me.desair.tus.server.upload.azure; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -import com.azure.storage.blob.BlobContainerClient; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import me.desair.tus.server.TestUtils; -import me.desair.tus.server.checksum.ChecksumAlgorithm; -import me.desair.tus.server.exception.MaxAppendSizeExceededException; -import me.desair.tus.server.upload.UploadId; -import me.desair.tus.server.upload.UploadInfo; -import org.junit.AfterClass; -import org.junit.Assume; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.testcontainers.containers.GenericContainer; - -public class AzureBlobStorageServiceTest { - - private static GenericContainer azuriteContainer; - - @BeforeClass - public static void setUpClass() { - Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers Azurite test", - TestUtils.isContainerRuntimeAvailable()); - azuriteContainer = TestUtils.createAzuriteContainer(); - azuriteContainer.start(); - } - - @AfterClass - public static void tearDownClass() { - if (azuriteContainer != null) { - azuriteContainer.stop(); - } - } - - private BlobContainerClient containerClient; - private AzureBlobStorageService storageService; - - @Before - public void setUp() { - Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - containerClient = - TestUtils.createBlobContainerClient( - azuriteContainer, "unit-test-container-" + System.nanoTime()); - storageService = new AzureBlobStorageService(containerClient); - } - - @Test - public void createShouldSetIdAndSaveInfo() throws Exception { - UploadInfo info = new UploadInfo(); - info.setLength(1000L); - - UploadInfo created = storageService.create(info, "owner1"); - - assertNotNull(created.getId()); - assertEquals("owner1", created.getOwnerKey()); - assertEquals(Long.valueOf(0L), created.getOffset()); - assertEquals("uploads/" + created.getId(), created.getStorageUploadId()); - } - - @Test - public void createZeroLengthUploadShouldCommitEmptyDataBlob() throws Exception { - UploadInfo info = new UploadInfo(); - info.setLength(0L); - - UploadInfo created = storageService.create(info, "owner1"); - - assertNotNull(created.getId()); - assertEquals(Long.valueOf(0L), created.getOffset()); - } - - @Test - public void getUploadInfoShouldReturnInfo() throws Exception { - UploadInfo info = new UploadInfo(); - info.setLength(500L); - UploadInfo created = storageService.create(info, "owner1"); - - UploadInfo fetched = storageService.getUploadInfo(created.getId()); - assertNotNull(fetched); - assertEquals(created.getId(), fetched.getId()); - assertEquals(Long.valueOf(500L), fetched.getLength()); - } - - @Test - public void getUploadInfoNotFoundShouldReturnNull() throws Exception { - UploadInfo fetched = storageService.getUploadInfo(new UploadId("non-existing-id")); - assertNull(fetched); - } - - @Test - public void getUploadInfoOwnerIsolation() throws Exception { - UploadInfo info = new UploadInfo(); - info.setLength(500L); - UploadInfo created = storageService.create(info, "owner1"); - - String url = "/test/upload/" + created.getId(); - assertNull(storageService.getUploadInfo(url, "wrong-owner")); - assertNotNull(storageService.getUploadInfo(url, "owner1")); - } - - @Test - public void maxAppendSizeFallback() { - storageService.setMaxUploadSize(5000L); - assertEquals(Long.valueOf(5000L), storageService.getMaxAppendSize()); - - storageService.setMaxAppendSize(2000L); - assertEquals(Long.valueOf(2000L), storageService.getMaxAppendSize()); - } - - @Test(expected = MaxAppendSizeExceededException.class) - public void appendExceedsMaxAppendSizeShouldThrow() throws Exception { - storageService.setMaxAppendSize(10L); - - UploadInfo info = new UploadInfo(); - info.setLength(100L); - UploadInfo created = storageService.create(info, "owner1"); - - ByteArrayInputStream bais = new ByteArrayInputStream("01234567890123456789".getBytes()); - storageService.append(created, bais); - } - - @Test - public void terminateUploadShouldDeleteBlobs() throws Exception { - UploadInfo info = new UploadInfo(); - info.setLength(1000L); - info.setChecksum("5d41402abc4b2a76b9719d911017c592"); - info.setChecksumAlgorithm(ChecksumAlgorithm.MD5); - - UploadInfo created = storageService.create(info, "owner1"); - - storageService.terminateUpload(created); - assertNull(storageService.getUploadInfo(created.getId())); - } - - @Test - public void getAzureBlobNameShouldReturnBlobName() throws Exception { - UploadInfo info = new UploadInfo(); - info.setLength(1000L); - UploadInfo created = storageService.create(info, "owner1"); - - String blobName = storageService.getAzureBlobName("/test/upload/" + created.getId(), "owner1"); - assertEquals("uploads/" + created.getId(), blobName); - } - - @Test - public void copyUploadToShouldCopyDataToOutputStream() throws Exception { - UploadInfo info = new UploadInfo(); - info.setLength(9L); - UploadInfo created = storageService.create(info, "owner1"); - - ByteArrayInputStream bais = new ByteArrayInputStream("test-data".getBytes()); - storageService.append(created, bais); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - storageService.copyUploadTo(created, baos); - - assertEquals("test-data", baos.toString()); - } - - @Test - public void getUploadedBytesShouldReturnInputStream() throws Exception { - UploadInfo info = new UploadInfo(); - info.setLength(9L); - UploadInfo created = storageService.create(info, "owner1"); - - ByteArrayInputStream bais = new ByteArrayInputStream("test-data".getBytes()); - storageService.append(created, bais); - - try (java.io.InputStream is = storageService.getUploadedBytes(created.getId())) { - assertNotNull(is); - assertEquals( - "test-data", - org.apache.commons.io.IOUtils.toString(is, java.nio.charset.StandardCharsets.UTF_8)); - } - } - - @Test - public void removeLastNumberOfBytesPartBlobOnly() throws Exception { - UploadInfo info = new UploadInfo(); - info.setLength(10L); - UploadInfo created = storageService.create(info, "owner1"); - - storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); - assertEquals(Long.valueOf(10L), created.getOffset()); - - storageService.removeLastNumberOfBytes(created, 3L); - assertEquals(Long.valueOf(7L), created.getOffset()); - } -} diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java deleted file mode 100644 index 869074ff..00000000 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java +++ /dev/null @@ -1,78 +0,0 @@ -package me.desair.tus.server.upload.azure; - -import static org.junit.Assert.assertEquals; - -import com.azure.storage.blob.BlobClient; -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.specialized.BlobLeaseClient; -import com.azure.storage.blob.specialized.BlobLeaseClientBuilder; -import me.desair.tus.server.TestUtils; -import me.desair.tus.server.upload.UploadLock; -import org.junit.AfterClass; -import org.junit.Assume; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.testcontainers.containers.GenericContainer; - -public class AzureBlobUploadLockTest { - - private static GenericContainer azuriteContainer; - - @BeforeClass - public static void setUpClass() { - Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers Azurite test", - TestUtils.isContainerRuntimeAvailable()); - azuriteContainer = TestUtils.createAzuriteContainer(); - azuriteContainer.start(); - } - - @AfterClass - public static void tearDownClass() { - if (azuriteContainer != null) { - azuriteContainer.stop(); - } - } - - private BlobContainerClient containerClient; - private String uploadUri; - private UploadLock uploadLock; - - @Before - public void setUp() throws Exception { - Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - containerClient = - TestUtils.createBlobContainerClient( - azuriteContainer, "lock-unit-container2-" + System.nanoTime()); - - uploadUri = "/test/upload/12345"; - BlobClient lockBlob = containerClient.getBlobClient("locks/12345.lock"); - lockBlob.getAppendBlobClient().create(); - BlobLeaseClient leaseClient = new BlobLeaseClientBuilder().blobClient(lockBlob).buildClient(); - leaseClient.acquireLease(30); - - uploadLock = new AzureBlobUploadLock(leaseClient, lockBlob, uploadUri); - } - - @Test - public void getUploadUriShouldReturnUri() { - assertEquals(uploadUri, uploadLock.getUploadUri()); - } - - @Test - public void releaseShouldReleaseLease() { - uploadLock.release(); - } - - @Test - public void closeShouldCallRelease() throws Exception { - uploadLock.close(); - } - - @Test - public void doubleReleaseShouldBeIdempotent() { - uploadLock.release(); - uploadLock.release(); - } -} diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobConcatenationService.java similarity index 57% rename from src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java rename to src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobConcatenationService.java index bed0323b..32fb8687 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobConcatenationService.java @@ -7,6 +7,7 @@ import com.azure.storage.blob.BlobContainerClient; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; import me.desair.tus.server.TestUtils; @@ -20,15 +21,13 @@ import org.junit.Test; import org.testcontainers.containers.GenericContainer; -public class AzureBlobConcatenationServiceTest { +public class ITAzureBlobConcatenationService { private static GenericContainer azuriteContainer; @BeforeClass public static void setUpClass() { - Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers Azurite test", - TestUtils.isContainerRuntimeAvailable()); + Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); azuriteContainer = TestUtils.createAzuriteContainer(); azuriteContainer.start(); } @@ -46,7 +45,7 @@ public static void tearDownClass() { @Before public void setUp() { - Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); + Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable() && azuriteContainer != null); containerClient = TestUtils.createBlobContainerClient( azuriteContainer, "concat-unit-container-" + System.nanoTime()); @@ -148,4 +147,98 @@ public void getPartialUploadsShouldReturnList() throws Exception { public void getPartialUploadsShouldReturnEmptyOnNullInfo() throws Exception { assertTrue(concatenationService.getPartialUploads(null).isEmpty()); } + + @Test + public void getPartialUploadsShouldReturnEmptyOnNullPartIds() throws Exception { + UploadInfo info = new UploadInfo(); + info.setConcatenationPartIds(null); + assertTrue(concatenationService.getPartialUploads(info).isEmpty()); + } + + @Test + public void mergeShouldUpdateExpirationWhenExpirationPeriodIsSet() throws Exception { + storageService.setUploadExpirationPeriod(5000L); + + UploadInfo part1Info = new UploadInfo(); + part1Info.setLength(5L); + UploadInfo part1 = storageService.create(part1Info, null); + storageService.append(part1, new ByteArrayInputStream("part1".getBytes())); + + UploadInfo finalInfo = new UploadInfo(); + finalInfo.setConcatenationPartIds(Arrays.asList("/test/upload/" + part1.getId())); + finalInfo.setUploadType(UploadType.CONCATENATED); + UploadInfo createdFinal = storageService.create(finalInfo, null); + + concatenationService.merge(createdFinal); + + assertNotNull(createdFinal.getExpirationTimestamp()); + assertTrue(createdFinal.getExpirationTimestamp() > System.currentTimeMillis()); + } + + @Test + public void mergeShouldDoNothingWhenIncompleteOrExpiredPartials() throws Exception { + UploadInfo part1Info = new UploadInfo(); + part1Info.setLength(10L); // length 10, but offset 0 (in progress) + UploadInfo part1 = storageService.create(part1Info, null); + + UploadInfo finalInfo = new UploadInfo(); + finalInfo.setConcatenationPartIds(Arrays.asList("/test/upload/" + part1.getId())); + finalInfo.setUploadType(UploadType.CONCATENATED); + UploadInfo createdFinal = storageService.create(finalInfo, null); + + concatenationService.merge(createdFinal); + // Should not merge since part1 is still in progress + assertEquals(Long.valueOf(0L), createdFinal.getOffset()); + } + + @Test + public void mergeShouldDoNothingWhenPartInfoLengthIsNull() throws Exception { + UploadInfo part1Info = new UploadInfo(); + UploadInfo part1 = storageService.create(part1Info, null); // length null + + UploadInfo finalInfo = new UploadInfo(); + finalInfo.setConcatenationPartIds(Arrays.asList("/test/upload/" + part1.getId())); + finalInfo.setUploadType(UploadType.CONCATENATED); + UploadInfo createdFinal = storageService.create(finalInfo, null); + + concatenationService.merge(createdFinal); + assertEquals(Long.valueOf(0L), createdFinal.getOffset()); + } + + @Test + public void mergeShouldDoNothingWhenFinalUploadNotInProgress() throws Exception { + UploadInfo finalInfo = new UploadInfo(); + finalInfo.setOffset(10L); + finalInfo.setLength(10L); // upload completed, not in progress + finalInfo.setConcatenationPartIds(Arrays.asList("/test/upload/some-id")); + + concatenationService.merge(finalInfo); + } + + @Test + public void getConcatenatedBytesShouldReturnBytesForCompletedUpload() throws Exception { + UploadInfo info = new UploadInfo(); + info.setOffset(5L); + info.setLength(5L); // completed (not in progress) + UploadInfo created = storageService.create(info, null); + storageService.append(created, new ByteArrayInputStream("hello".getBytes())); + + InputStream is = concatenationService.getConcatenatedBytes(created); + assertNotNull(is); + assertEquals("hello", org.apache.commons.io.IOUtils.toString(is, StandardCharsets.UTF_8)); + } + + @Test + public void constructorPrefixSanitizationVariants() { + AzureBlobConcatenationService service1 = + new AzureBlobConcatenationService(containerClient, null, storageService); + AzureBlobConcatenationService service2 = + new AzureBlobConcatenationService(containerClient, "/custom/prefix", storageService); + AzureBlobConcatenationService service3 = + new AzureBlobConcatenationService(containerClient, "custom/prefix/", storageService); + + assertNotNull(service1); + assertNotNull(service2); + assertNotNull(service3); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java new file mode 100644 index 00000000..a54d0a94 --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java @@ -0,0 +1,210 @@ +package me.desair.tus.server.upload.azure; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import com.azure.storage.blob.BlobContainerClient; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import me.desair.tus.server.TestUtils; +import me.desair.tus.server.exception.UploadAlreadyLockedException; +import me.desair.tus.server.upload.TimeBasedUploadIdFactory; +import me.desair.tus.server.upload.UploadId; +import me.desair.tus.server.upload.UploadLock; +import me.desair.tus.server.util.InterruptibleInputStream; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.testcontainers.containers.GenericContainer; + +public class ITAzureBlobLockingService { + + private static GenericContainer azuriteContainer; + + @BeforeClass + public static void setUpClass() { + Assume.assumeTrue( + "Container runtime is not available; skipping Testcontainers Azurite test", + TestUtils.isContainerRuntimeAvailable()); + azuriteContainer = TestUtils.createAzuriteContainer(); + azuriteContainer.start(); + } + + @AfterClass + public static void tearDownClass() { + if (azuriteContainer != null) { + azuriteContainer.stop(); + } + } + + private BlobContainerClient containerClient; + private AzureBlobLockingService lockingService; + + @Before + public void setUp() { + Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); + containerClient = + TestUtils.createBlobContainerClient( + azuriteContainer, "lock-unit-container-" + System.nanoTime()); + lockingService = new AzureBlobLockingService(containerClient); + TimeBasedUploadIdFactory idFactory = new TimeBasedUploadIdFactory(); + idFactory.setUploadUri("/test/upload"); + lockingService.setIdFactory(idFactory); + } + + @Test + public void lockUploadByUriShouldReturnNullOnInvalidUri() throws Exception { + assertNull(lockingService.lockUploadByUri("invalid-uri-no-id")); + } + + @Test + public void isLockedShouldReturnFalseWhenLockBlobDoesNotExist() { + assertFalse(lockingService.isLocked(new UploadId("12345"))); + } + + @Test + public void lockUploadByUriShouldAcquireLock() throws Exception { + UploadLock lock = lockingService.lockUploadByUri("/test/upload/12345"); + assertNotNull(lock); + assertEquals("/test/upload/12345", lock.getUploadUri()); + assertTrue(lockingService.isLocked(new UploadId("12345"))); + lock.release(); + } + + @Test(expected = UploadAlreadyLockedException.class) + public void lockUploadByUriShouldThrowOnLockContention() throws Exception { + UploadLock lock1 = lockingService.lockUploadByUri("/test/upload/12345"); + assertNotNull(lock1); + try { + lockingService.lockUploadByUri("/test/upload/12345"); + } finally { + lock1.release(); + } + } + + @Test + public void registerInputStreamAndRequestReleaseShouldInterruptStream() throws Exception { + ByteArrayInputStream bais = new ByteArrayInputStream("data".getBytes()); + InterruptibleInputStream stream = new InterruptibleInputStream(bais); + + lockingService.registerInputStream("/test/upload/12345", stream); + lockingService.requestLockRelease("/test/upload/12345"); + + try { + stream.read(); + } catch (Exception e) { + assertNotNull(e); + } + } + + @Test + public void cleanupStaleLocksShouldNotThrow() throws Exception { + lockingService.cleanupStaleLocks(); + } + + @Test + public void closeShouldCleanUpResources() throws Exception { + lockingService.close(); + lockingService.close(); + } + + @Test(expected = NullPointerException.class) + public void constructorShouldThrowOnNullContainerClient() { + new AzureBlobLockingService(null); + } + + @Test(expected = NullPointerException.class) + public void setIdFactoryShouldThrowOnNull() { + lockingService.setIdFactory(null); + } + + @Test + public void isLockedShouldReturnFalseForNullId() { + assertFalse(lockingService.isLocked(null)); + } + + @Test + public void isLockedShouldReturnFalseWhenPropertiesThrowException() { + assertFalse(lockingService.isLocked(new UploadId("non-existent-lock-id-999"))); + } + + @Test + public void registerInputStreamShouldDoNothingOnInvalidUriOrStandardStream() { + ByteArrayInputStream bais = new ByteArrayInputStream("test".getBytes()); + + lockingService.registerInputStream("invalid-uri", bais); + lockingService.registerInputStream("/test/upload/12345", bais); + } + + @Test + public void requestLockReleaseShouldDoNothingOnInvalidUri() { + lockingService.requestLockRelease("invalid-uri"); + } + + @Test + public void prefixSanitizationVariants() { + AzureBlobLockingService service1 = new AzureBlobLockingService(containerClient, null); + AzureBlobLockingService service2 = + new AzureBlobLockingService(containerClient, "/custom/locks"); + AzureBlobLockingService service3 = + new AzureBlobLockingService(containerClient, "custom/locks/"); + + assertNotNull(service1); + assertNotNull(service2); + assertNotNull(service3); + } + + @Test + public void watchdogPollingDetectsStopSignalBlob() throws Exception { + ByteArrayInputStream bais = new ByteArrayInputStream("data".getBytes()); + InterruptibleInputStream stream = new InterruptibleInputStream(bais); + + lockingService.registerInputStream("/test/upload/54321", stream); + + com.azure.storage.blob.BlobClient stopBlob = containerClient.getBlobClient("locks/54321.stop"); + stopBlob.upload(com.azure.core.util.BinaryData.fromString("stop"), true); + + long deadline = System.currentTimeMillis() + 3500L; + while (!stream.isInterrupted() && System.currentTimeMillis() < deadline) { + Thread.sleep(100L); + } + + assertTrue("Expected stream to be interrupted by watchdog thread", stream.isInterrupted()); + assertFalse("Expected .stop blob to be deleted by watchdog thread", stopBlob.exists()); + } + + @Test + public void ensureLockBlobExistsHandlesExceptions() { + com.azure.storage.blob.BlobClient lockBlob = + containerClient.getBlobClient("locks/nonexistentcontainer/invalid.lock"); + lockingService.ensureLockBlobExists(lockBlob); + } + + @Test + public void closeInterruptsActiveWatchdogThread() throws Exception { + ByteArrayInputStream bais = new ByteArrayInputStream("data".getBytes()); + InterruptibleInputStream stream = new InterruptibleInputStream(bais); + + lockingService.registerInputStream("/test/upload/88888", stream); + lockingService.close(); + } + + @Test(expected = IOException.class) + public void lockUploadByUriShouldThrowIOExceptionOnStorageException() throws Exception { + com.azure.storage.blob.BlobServiceClient serviceClient = containerClient.getServiceClient(); + BlobContainerClient nonExistentContainer = + serviceClient.getBlobContainerClient( + "non-existent-container-" + System.currentTimeMillis()); + AzureBlobLockingService service = new AzureBlobLockingService(nonExistentContainer); + TimeBasedUploadIdFactory idFactory = new TimeBasedUploadIdFactory(); + idFactory.setUploadUri("/test/upload"); + service.setIdFactory(idFactory); + + service.lockUploadByUri("/test/upload/12345"); + } +} diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java new file mode 100644 index 00000000..d6555ea3 --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java @@ -0,0 +1,496 @@ +package me.desair.tus.server.upload.azure; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import com.azure.storage.blob.BlobContainerClient; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import me.desair.tus.server.TestUtils; +import me.desair.tus.server.checksum.ChecksumAlgorithm; +import me.desair.tus.server.exception.MaxAppendSizeExceededException; +import me.desair.tus.server.exception.MinAppendSizeNotMetException; +import me.desair.tus.server.exception.UploadNotFoundException; +import me.desair.tus.server.upload.UploadId; +import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLock; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.testcontainers.containers.GenericContainer; + +public class ITAzureBlobStorageService { + + private static GenericContainer azuriteContainer; + + @BeforeClass + public static void setUpClass() { + Assume.assumeTrue( + "Container runtime is not available; skipping Testcontainers Azurite test", + TestUtils.isContainerRuntimeAvailable()); + azuriteContainer = TestUtils.createAzuriteContainer(); + azuriteContainer.start(); + } + + @AfterClass + public static void tearDownClass() { + if (azuriteContainer != null) { + azuriteContainer.stop(); + } + } + + private BlobContainerClient containerClient; + private AzureBlobStorageService storageService; + + @Before + public void setUp() { + Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable() && azuriteContainer != null); + containerClient = + TestUtils.createBlobContainerClient( + azuriteContainer, "unit-test-container-" + System.nanoTime()); + storageService = new AzureBlobStorageService(containerClient); + } + + @Test + public void configurationGettersAndSetters() { + storageService.setMaxAppendSize(500L); + assertEquals(Long.valueOf(500L), storageService.getMaxAppendSize()); + + storageService.setMinAppendSize(100L); + assertEquals(Long.valueOf(100L), storageService.getMinAppendSize()); + + storageService.setPreferredBlockSize(8L * 1024 * 1024); + assertEquals(8L * 1024 * 1024, storageService.getPreferredBlockSize()); + + storageService.setUploadDeduplicationEnabled(true); + assertTrue(storageService.isUploadDeduplicationEnabled()); + + storageService.setUploadExpirationPeriod(3600000L); + assertEquals(Long.valueOf(3600000L), storageService.getUploadExpirationPeriod()); + } + + @Test + public void constructorCustomPrefixes() { + AzureBlobStorageService customService = + new AzureBlobStorageService( + containerClient, + "custom-uploads/", + "custom-metadata/", + "custom-checksums/", + "custom-parts/", + java.nio.file.Paths.get(System.getProperty("java.io.tmpdir"))); + assertNotNull(customService); + } + + @Test + public void getAzureBlobNameShouldReturnBlobName() throws Exception { + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("12345")); + assertEquals("uploads/12345", storageService.getAzureBlobName(info)); + + info.setDuplicatesUploadId(new UploadId("parent-999")); + assertEquals("uploads/parent-999", storageService.getAzureBlobName(info)); + } + + @Test(expected = IllegalArgumentException.class) + public void setPreferredBlockSizeTooSmallShouldThrow() { + storageService.setPreferredBlockSize(1024L); // less than 4MB + } + + @Test(expected = IllegalArgumentException.class) + public void setPreferredBlockSizeTooLargeShouldThrow() { + storageService.setPreferredBlockSize(5000L * 1024 * 1024); // greater than 4000MB + } + + @Test(expected = MinAppendSizeNotMetException.class) + public void minAppendSizeNotMetShouldThrow() throws Exception { + storageService.setMinAppendSize(100L); + + UploadInfo info = new UploadInfo(); + info.setLength(500L); + UploadInfo created = storageService.create(info, "owner1"); + + storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + } + + @Test(expected = MaxAppendSizeExceededException.class) + public void appendExceedsMaxAppendSizeShouldThrow() throws Exception { + storageService.setMaxAppendSize(5L); + + UploadInfo info = new UploadInfo(); + info.setLength(500L); + UploadInfo created = storageService.create(info, "owner1"); + + storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + } + + @Test(expected = NullPointerException.class) + public void removeLastNumberOfBytesShouldThrowOnNullInfo() throws Exception { + storageService.removeLastNumberOfBytes(null, 10L); + } + + @Test(expected = UploadNotFoundException.class) + public void getUploadedBytesByUriNotFoundShouldThrow() throws Exception { + storageService.getUploadedBytes("/test/upload/non-existent", "owner1"); + } + + @Test + public void getUploadedBytesNullIdShouldReturnNull() throws Exception { + assertNull(storageService.getUploadedBytes((UploadId) null)); + } + + @Test(expected = UploadNotFoundException.class) + public void copyUploadToNotFoundShouldThrow() throws Exception { + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("non-existent-id")); + storageService.copyUploadTo(info, new ByteArrayOutputStream()); + } + + @Test(expected = NullPointerException.class) + public void copyUploadToNullInfoShouldThrow() throws Exception { + storageService.copyUploadTo(null, new ByteArrayOutputStream()); + } + + @Test(expected = NullPointerException.class) + public void copyUploadToNullStreamShouldThrow() throws Exception { + storageService.copyUploadTo(new UploadInfo(), null); + } + + @Test + public void terminateUploadNullInfoShouldDoNothing() throws Exception { + storageService.terminateUpload(null); + storageService.terminateUpload(new UploadInfo()); + } + + @Test + public void getAzureBlobNameNullInfoShouldReturnNull() throws Exception { + assertNull(storageService.getAzureBlobName((UploadInfo) null)); + assertNull(storageService.getAzureBlobName("/test/upload/invalid", "owner1")); + } + + @Test + public void getUploadInfoByChecksumDisabledOrNull() throws Exception { + storageService.setUploadDeduplicationEnabled(false); + assertNull(storageService.getUploadInfoByChecksum("checksum", ChecksumAlgorithm.MD5)); + + storageService.setUploadDeduplicationEnabled(true); + assertNull(storageService.getUploadInfoByChecksum(null, ChecksumAlgorithm.MD5)); + assertNull(storageService.getUploadInfoByChecksum("checksum", null)); + } + + @Test + public void createZeroLengthUploadShouldCommitEmptyDataBlob() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(0L); + UploadInfo created = storageService.create(info, "owner1"); + + assertNotNull(created.getId()); + assertEquals(Long.valueOf(0L), created.getOffset()); + + UploadInfo fetched = storageService.getUploadInfo(created.getId()); + assertNotNull(fetched); + assertEquals(Long.valueOf(0L), fetched.getOffset()); + } + + @Test + public void appendConsecutiveSubThresholdChunksToPartBlob() throws Exception { + storageService.setPreferredBlockSize(4L * 1024 * 1024); // 4MB + + UploadInfo info = new UploadInfo(); + info.setLength(100L); + UploadInfo created = storageService.create(info, "owner1"); + + storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + assertEquals(Long.valueOf(10L), created.getOffset()); + + storageService.append(created, new ByteArrayInputStream("abcdefghij".getBytes())); + assertEquals(Long.valueOf(20L), created.getOffset()); + + try (InputStream is = storageService.getUploadedBytes(created.getId())) { + assertEquals( + "0123456789abcdefghij", + org.apache.commons.io.IOUtils.toString(is, StandardCharsets.UTF_8)); + } + } + + @Test + public void maxAppendSizeFallback() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(100L); + UploadInfo created = storageService.create(info, "owner1"); + storageService.setMaxAppendSize(null); // disable limit + + storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + assertEquals(Long.valueOf(10L), created.getOffset()); + } + + @Test + public void getUploadedBytesShouldReturnInputStream() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(12L); + UploadInfo created = storageService.create(info, "owner1"); + + storageService.append(created, new ByteArrayInputStream("Hello World!".getBytes())); + + try (InputStream is = storageService.getUploadedBytes(created.getId())) { + assertNotNull(is); + assertEquals( + "Hello World!", org.apache.commons.io.IOUtils.toString(is, StandardCharsets.UTF_8)); + } + + try (InputStream is = + storageService.getUploadedBytes("/test/upload/" + created.getId(), "owner1")) { + assertNotNull(is); + assertEquals( + "Hello World!", org.apache.commons.io.IOUtils.toString(is, StandardCharsets.UTF_8)); + } + } + + @Test + public void getUploadInfoShouldReturnInfo() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(100L); + UploadInfo created = storageService.create(info, "owner1"); + + UploadInfo fetchedById = storageService.getUploadInfo(created.getId()); + assertNotNull(fetchedById); + assertEquals(created.getId(), fetchedById.getId()); + + UploadInfo fetchedByUri = + storageService.getUploadInfo("/test/upload/" + created.getId(), "owner1"); + assertNotNull(fetchedByUri); + assertEquals(created.getId(), fetchedByUri.getId()); + + assertNull(storageService.getUploadInfo(new UploadId("non-existing-id"))); + assertNull(storageService.getUploadInfo("/test/upload/non-existing-id", "owner1")); + } + + @Test + public void copyUploadToShouldCopyDataToOutputStream() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(11L); + UploadInfo created = storageService.create(info, "owner1"); + + storageService.append(created, new ByteArrayInputStream("Hello Azure".getBytes())); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + storageService.copyUploadTo(created, baos); + assertEquals("Hello Azure", baos.toString()); + + ByteArrayOutputStream baosUri = new ByteArrayOutputStream(); + UploadInfo fetchedUri = + storageService.getUploadInfo("/test/upload/" + created.getId(), "owner1"); + storageService.copyUploadTo(fetchedUri, baosUri); + assertEquals("Hello Azure", baosUri.toString()); + } + + @Test + public void terminateUploadShouldDeleteBlobs() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(10L); + UploadInfo created = storageService.create(info, "owner1"); + + storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + assertNotNull(storageService.getUploadInfo(created.getId())); + + storageService.terminateUpload(created); + assertNull(storageService.getUploadInfo(created.getId())); + } + + @Test + public void removeLastNumberOfBytesPartBlobOnly() throws Exception { + storageService.setPreferredBlockSize(4L * 1024 * 1024); + + UploadInfo info = new UploadInfo(); + info.setLength(100L); + UploadInfo created = storageService.create(info, "owner1"); + + storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + assertEquals(Long.valueOf(10L), created.getOffset()); + + // Remove 4 bytes from sub-threshold part blob + storageService.removeLastNumberOfBytes(created, 4L); + assertEquals(Long.valueOf(6L), created.getOffset()); + + try (InputStream is = storageService.getUploadedBytes(created.getId())) { + assertNotNull(is); + assertEquals("012345", org.apache.commons.io.IOUtils.toString(is, StandardCharsets.UTF_8)); + } + + // Remove remaining bytes + storageService.removeLastNumberOfBytes(created, 6L); + assertEquals(Long.valueOf(0L), created.getOffset()); + } + + @Test + public void removeLastNumberOfBytesTrimPartBlobWithCommittedBlocks() throws Exception { + storageService.setPreferredBlockSize(4L * 1024 * 1024); // 4MB blocks + + byte[] data = new byte[9 * 1024 * 1024]; + java.util.Arrays.fill(data, (byte) 'B'); + + UploadInfo info = new UploadInfo(); + info.setLength((long) data.length); + UploadInfo created = storageService.create(info, "owner1"); + + storageService.append(created, new ByteArrayInputStream(data)); + assertEquals(Long.valueOf(data.length), created.getOffset()); + + // Truncate by 500KB (targetOffset 8.5MB > 8MB block blob size) + storageService.removeLastNumberOfBytes(created, 500 * 1024); + assertEquals(Long.valueOf(data.length - 500 * 1024), created.getOffset()); + + try (InputStream is = storageService.getUploadedBytes(created.getId())) { + assertNotNull(is); + byte[] readBytes = org.apache.commons.io.IOUtils.toByteArray(is); + assertEquals(data.length - 500 * 1024, readBytes.length); + } + } + + @Test + public void appendMultiBlockPayloadAndTruncateCommittedBlocks() throws Exception { + storageService.setPreferredBlockSize(4L * 1024 * 1024); // 4MB blocks + + byte[] data = new byte[9 * 1024 * 1024]; + java.util.Arrays.fill(data, (byte) 'A'); + + UploadInfo info = new UploadInfo(); + info.setLength((long) data.length); + UploadInfo created = storageService.create(info, "owner1"); + + storageService.append(created, new ByteArrayInputStream(data)); + assertEquals(Long.valueOf(data.length), created.getOffset()); + + // Truncate by 2MB (targetOffset 7MB <= 8MB block blob size) + storageService.removeLastNumberOfBytes(created, 2L * 1024 * 1024); + assertEquals(Long.valueOf(7L * 1024 * 1024), created.getOffset()); + + try (InputStream is = storageService.getUploadedBytes(created.getId())) { + assertNotNull(is); + } + } + + @Test + public void cleanupExpiredUploadsShouldDeleteExpired() throws Exception { + storageService.setUploadExpirationPeriod(1L); + + UploadInfo info = new UploadInfo(); + info.setLength(10L); + UploadInfo created = storageService.create(info, "owner1"); + + storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + + Thread.sleep(50L); + // Cleanup with expiration period 0 (all uploads expired) + storageService.cleanupExpiredUploads(new AzureBlobLockingService(containerClient)); + assertNull(storageService.getUploadInfo(created.getId())); + } + + @Test + public void getUploadInfoByChecksumSelfCleaningStaleIndex() throws Exception { + storageService.setUploadDeduplicationEnabled(true); + + UploadInfo info = new UploadInfo(); + info.setLength(5L); + info.setChecksum("5d41402abc4b2a76b9719d911017c592"); + info.setChecksumAlgorithm(ChecksumAlgorithm.MD5); + + UploadInfo created = storageService.create(info, "owner1"); + storageService.append(created, new ByteArrayInputStream("hello".getBytes())); + + UploadInfo found = + storageService.getUploadInfoByChecksum( + "5d41402abc4b2a76b9719d911017c592", ChecksumAlgorithm.MD5); + assertNotNull(found); + assertEquals(created.getId(), found.getId()); + + // Manually delete upload blobs to simulate stale index + storageService.terminateUpload(created); + + // Stale index lookup should self-clean and return null + assertNull( + storageService.getUploadInfoByChecksum( + "5d41402abc4b2a76b9719d911017c592", ChecksumAlgorithm.MD5)); + } + + @Test + public void testFullUploadLifecycleOnAzurite() throws Exception { + AzureBlobStorageService storage = new AzureBlobStorageService(containerClient); + AzureBlobLockingService locking = new AzureBlobLockingService(containerClient); + + UploadInfo info = new UploadInfo(); + info.setLength(11L); + + UploadInfo created = storage.create(info, "owner1"); + assertNotNull(created.getId()); + + try (UploadLock lock = locking.lockUploadByUri("/test/upload/" + created.getId())) { + assertNotNull(lock); + storage.append(created, new ByteArrayInputStream("hello ".getBytes())); + storage.append(created, new ByteArrayInputStream("world".getBytes())); + } + + UploadInfo fetched = storage.getUploadInfo(created.getId()); + assertNotNull(fetched); + assertEquals(Long.valueOf(11L), fetched.getOffset()); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + storage.copyUploadTo(fetched, baos); + assertEquals("hello world", baos.toString()); + + String blobName = storage.getAzureBlobName("/test/upload/" + created.getId(), "owner1"); + assertEquals("uploads/" + created.getId(), blobName); + } + + @Test + public void testTruncateBytesOnAzurite() throws Exception { + AzureBlobStorageService storage = new AzureBlobStorageService(containerClient); + + UploadInfo info = new UploadInfo(); + info.setLength(10L); + UploadInfo created = storage.create(info, "owner1"); + + storage.append(created, new ByteArrayInputStream("0123456789".getBytes())); + assertEquals(Long.valueOf(10L), created.getOffset()); + + storage.removeLastNumberOfBytes(created, 3L); + assertEquals(Long.valueOf(7L), created.getOffset()); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + storage.copyUploadTo(created, baos); + assertEquals("0123456", baos.toString()); + } + + @Test + public void testDeduplicationOnAzurite() throws Exception { + AzureBlobStorageService storage = new AzureBlobStorageService(containerClient); + storage.setUploadDeduplicationEnabled(true); + + UploadInfo parent = new UploadInfo(); + parent.setLength(5L); + parent.setChecksum("5d41402abc4b2a76b9719d911017c592"); + parent.setChecksumAlgorithm(ChecksumAlgorithm.MD5); + parent = storage.create(parent, "owner1"); + storage.append(parent, new ByteArrayInputStream("hello".getBytes())); + + UploadInfo child = new UploadInfo(); + child.setLength(5L); + child.setChecksum("5d41402abc4b2a76b9719d911017c592"); + child.setChecksumAlgorithm(ChecksumAlgorithm.MD5); + child = storage.create(child, "owner1"); + storage.append(child, new ByteArrayInputStream("hello".getBytes())); + + assertNotNull(child.getDuplicatesUploadId()); + assertEquals(parent.getId(), child.getDuplicatesUploadId()); + + storage.terminateUpload(parent); + assertNull( + storage.getUploadInfoByChecksum("5d41402abc4b2a76b9719d911017c592", ChecksumAlgorithm.MD5)); + } +} diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageServiceTest.java deleted file mode 100644 index 3f5de690..00000000 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageServiceTest.java +++ /dev/null @@ -1,128 +0,0 @@ -package me.desair.tus.server.upload.azure; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -import com.azure.storage.blob.BlobContainerClient; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import me.desair.tus.server.TestUtils; -import me.desair.tus.server.checksum.ChecksumAlgorithm; -import me.desair.tus.server.upload.UploadInfo; -import me.desair.tus.server.upload.UploadLock; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.testcontainers.containers.GenericContainer; - -public class ITAzureBlobStorageServiceTest { - - private static GenericContainer azurite; - private static BlobContainerClient containerClient; - private static final String CONTAINER = "test-storage-azure-container"; - - @BeforeClass - public static void setUpClass() { - org.junit.Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers Azurite test", - TestUtils.isContainerRuntimeAvailable()); - - azurite = TestUtils.createAzuriteContainer(); - azurite.start(); - - containerClient = TestUtils.createBlobContainerClient(azurite, CONTAINER); - } - - @AfterClass - public static void tearDownClass() { - if (azurite != null) { - azurite.stop(); - } - } - - @Test - public void testFullUploadLifecycleOnAzurite() throws Exception { - org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - - AzureBlobStorageService storage = new AzureBlobStorageService(containerClient); - AzureBlobLockingService locking = new AzureBlobLockingService(containerClient); - - UploadInfo info = new UploadInfo(); - info.setLength(11L); - - UploadInfo created = storage.create(info, "owner1"); - assertNotNull(created.getId()); - - try (UploadLock lock = locking.lockUploadByUri("/test/upload/" + created.getId())) { - assertNotNull(lock); - storage.append(created, new ByteArrayInputStream("hello ".getBytes())); - storage.append(created, new ByteArrayInputStream("world".getBytes())); - } - - UploadInfo fetched = storage.getUploadInfo(created.getId()); - assertNotNull(fetched); - assertEquals(Long.valueOf(11L), fetched.getOffset()); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - storage.copyUploadTo(fetched, baos); - assertEquals("hello world", baos.toString()); - - String blobName = storage.getAzureBlobName("/test/upload/" + created.getId(), "owner1"); - assertEquals("uploads/" + created.getId(), blobName); - } - - @Test - public void testTruncateBytesOnAzurite() throws Exception { - org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - - AzureBlobStorageService storage = new AzureBlobStorageService(containerClient); - - UploadInfo info = new UploadInfo(); - info.setLength(10L); - UploadInfo created = storage.create(info, "owner1"); - - storage.append(created, new ByteArrayInputStream("0123456789".getBytes())); - assertEquals(Long.valueOf(10L), created.getOffset()); - - // Truncate 3 bytes - storage.removeLastNumberOfBytes(created, 3L); - assertEquals(Long.valueOf(7L), created.getOffset()); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - storage.copyUploadTo(created, baos); - assertEquals("0123456", baos.toString()); - } - - @Test - public void testDeduplicationOnAzurite() throws Exception { - org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - - AzureBlobStorageService storage = new AzureBlobStorageService(containerClient); - storage.setUploadDeduplicationEnabled(true); - - // Parent upload - UploadInfo parent = new UploadInfo(); - parent.setLength(5L); - parent.setChecksum("5d41402abc4b2a76b9719d911017c592"); - parent.setChecksumAlgorithm(ChecksumAlgorithm.MD5); - parent = storage.create(parent, "owner1"); - storage.append(parent, new ByteArrayInputStream("hello".getBytes())); - - // Child upload matching parent checksum - UploadInfo child = new UploadInfo(); - child.setLength(5L); - child.setChecksum("5d41402abc4b2a76b9719d911017c592"); - child.setChecksumAlgorithm(ChecksumAlgorithm.MD5); - child = storage.create(child, "owner1"); - storage.append(child, new ByteArrayInputStream("hello".getBytes())); - - assertNotNull(child.getDuplicatesUploadId()); - assertEquals(parent.getId(), child.getDuplicatesUploadId()); - - // Clean up parent - storage.terminateUpload(parent); - assertNull( - storage.getUploadInfoByChecksum("5d41402abc4b2a76b9719d911017c592", ChecksumAlgorithm.MD5)); - } -} diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobUploadLock.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobUploadLock.java new file mode 100644 index 00000000..021f8c3b --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobUploadLock.java @@ -0,0 +1,132 @@ +package me.desair.tus.server.upload.azure; + +import static org.junit.Assert.assertEquals; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.specialized.BlobLeaseClient; +import com.azure.storage.blob.specialized.BlobLeaseClientBuilder; +import me.desair.tus.server.TestUtils; +import me.desair.tus.server.upload.UploadLock; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.testcontainers.containers.GenericContainer; + +public class ITAzureBlobUploadLock { + + private static GenericContainer azuriteContainer; + + @BeforeClass + public static void setUpClass() { + Assume.assumeTrue( + "Container runtime is not available; skipping Testcontainers Azurite test", + TestUtils.isContainerRuntimeAvailable()); + azuriteContainer = TestUtils.createAzuriteContainer(); + azuriteContainer.start(); + } + + @AfterClass + public static void tearDownClass() { + if (azuriteContainer != null) { + azuriteContainer.stop(); + } + } + + private BlobContainerClient containerClient; + private String uploadUri; + private UploadLock uploadLock; + + @Before + public void setUp() throws Exception { + Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); + containerClient = + TestUtils.createBlobContainerClient( + azuriteContainer, "lock-unit-container2-" + System.nanoTime()); + + uploadUri = "/test/upload/12345"; + BlobClient lockBlob = containerClient.getBlobClient("locks/12345.lock"); + lockBlob.getAppendBlobClient().create(); + BlobLeaseClient leaseClient = new BlobLeaseClientBuilder().blobClient(lockBlob).buildClient(); + leaseClient.acquireLease(30); + + uploadLock = new AzureBlobUploadLock(leaseClient, lockBlob, uploadUri); + } + + @Test + public void getUploadUriShouldReturnUri() { + assertEquals(uploadUri, uploadLock.getUploadUri()); + } + + @Test + public void releaseShouldReleaseLease() { + uploadLock.release(); + } + + @Test + public void closeShouldCallRelease() throws Exception { + uploadLock.close(); + } + + @Test + public void doubleReleaseShouldBeIdempotent() { + uploadLock.release(); + uploadLock.release(); + } + + @Test(expected = NullPointerException.class) + public void constructorShouldThrowOnNullLeaseClient() { + BlobClient lockBlob = containerClient.getBlobClient("locks/nulltest.lock"); + new AzureBlobUploadLock(null, lockBlob, "/uri"); + } + + @Test(expected = NullPointerException.class) + public void constructorShouldThrowOnNullLockBlob() { + BlobClient lockBlob = containerClient.getBlobClient("locks/nulltest2.lock"); + lockBlob.getAppendBlobClient().create(); + BlobLeaseClient leaseClient = new BlobLeaseClientBuilder().blobClient(lockBlob).buildClient(); + new AzureBlobUploadLock(leaseClient, null, "/uri"); + } + + @Test(expected = NullPointerException.class) + public void constructorShouldThrowOnNullUploadUri() { + BlobClient lockBlob = containerClient.getBlobClient("locks/nulltest3.lock"); + lockBlob.getAppendBlobClient().create(); + BlobLeaseClient leaseClient = new BlobLeaseClientBuilder().blobClient(lockBlob).buildClient(); + new AzureBlobUploadLock(leaseClient, lockBlob, null); + } + + @Test + public void releaseShouldHandleReleaseLeaseExceptionGracefully() { + BlobClient lockBlob = containerClient.getBlobClient("locks/releasetest.lock"); + lockBlob.getAppendBlobClient().create(); + BlobLeaseClient leaseClient = new BlobLeaseClientBuilder().blobClient(lockBlob).buildClient(); + AzureBlobUploadLock lock = new AzureBlobUploadLock(leaseClient, lockBlob, "/test/uri"); + lock.release(); + } + + @Test + public void renewLeaseWhenReleasedShouldReturnImmediately() { + AzureBlobUploadLock lock = (AzureBlobUploadLock) uploadLock; + lock.release(); + lock.renewLease(); + } + + @Test + public void renewLeaseShouldRenewActiveLease() { + AzureBlobUploadLock lock = (AzureBlobUploadLock) uploadLock; + lock.renewLease(); + } + + @Test + public void renewLeaseFailureShouldCatchExceptionAndSetReleased() { + BlobClient lockBlob = containerClient.getBlobClient("locks/renewfail.lock"); + lockBlob.getAppendBlobClient().create(); + BlobLeaseClient leaseClient = new BlobLeaseClientBuilder().blobClient(lockBlob).buildClient(); + AzureBlobUploadLock lock = new AzureBlobUploadLock(leaseClient, lockBlob, "/test/uri"); + lock.renewLease(); + lock.renewLease(); + } +} From a578c15ce555a1466b582336ea4688f87478461c Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Wed, 12 Aug 2026 09:31:06 +0200 Subject: [PATCH 06/14] build: activate JDK 24+ JVM args via profile for final field mutation and native access --- pom.xml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 6b1897c9..251656be 100644 --- a/pom.xml +++ b/pom.xml @@ -22,6 +22,10 @@ 17 95 + + + + @@ -277,7 +281,7 @@ 3.2.5 - ${surefireArgLine} --enable-final-field-mutation=ALL-UNNAMED --enable-native-access=ALL-UNNAMED + ${surefireArgLine} ${surefireJvmArgs} ${project.build.directory}/surefire-reports @@ -296,7 +300,7 @@ 3.2.5 - ${failsafeArgLine} --enable-final-field-mutation=ALL-UNNAMED --enable-native-access=ALL-UNNAMED + ${failsafeArgLine} ${failsafeJvmArgs} ${project.build.directory}/surefire-reports @@ -331,6 +335,16 @@ + + jdk-24-plus + + [24,) + + + --enable-final-field-mutation=ALL-UNNAMED --enable-native-access=ALL-UNNAMED + --enable-final-field-mutation=ALL-UNNAMED --enable-native-access=ALL-UNNAMED + + skiptests From 68780292036e03c2f447e50f28c850a25097849f Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Wed, 12 Aug 2026 09:33:33 +0200 Subject: [PATCH 07/14] build: remove jdk-24-plus profile from pom.xml to maintain standard CI compatibility --- pom.xml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pom.xml b/pom.xml index 251656be..24da5b3f 100644 --- a/pom.xml +++ b/pom.xml @@ -335,16 +335,6 @@ - - jdk-24-plus - - [24,) - - - --enable-final-field-mutation=ALL-UNNAMED --enable-native-access=ALL-UNNAMED - --enable-final-field-mutation=ALL-UNNAMED --enable-native-access=ALL-UNNAMED - - skiptests From 88c05035688ca6117e66dc23934bf93ecf462400 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Wed, 12 Aug 2026 09:37:23 +0200 Subject: [PATCH 08/14] build: replace javac source/target options with release 17 to fix compiler warning --- pom.xml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 24da5b3f..8c26f9bd 100644 --- a/pom.xml +++ b/pom.xml @@ -18,8 +18,7 @@ UTF-8 - 17 - 17 + 17 95 @@ -77,12 +76,6 @@ 4.2.17.Final provided - - io.netty - netty-codec - 4.2.17.Final - provided - io.netty netty-codec-http @@ -326,7 +319,7 @@ maven-compiler-plugin 3.13.0 - 17 + 17 UTF-8 From 2f118ce5ee0cfbe812b0f47f5250f532bfe953ed Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Wed, 12 Aug 2026 10:02:38 +0200 Subject: [PATCH 09/14] fix(locking): enhance InterruptibleInputStream safety, avoid reflection in tests, and update AGENTS.md --- AGENTS.md | 3 + .../upload/azure/AzureBlobLockingService.java | 3 +- .../upload/disk/DiskLockingService.java | 53 ++++--- .../server/upload/s3/S3LockingService.java | 14 +- .../java/me/desair/tus/server/util/Utils.java | 22 +++ .../upload/disk/DiskLockingServiceTest.java | 137 ++++++++++-------- .../me/desair/tus/server/util/UtilsTest.java | 31 ++++ 7 files changed, 163 insertions(+), 100 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 457f1ace..aaf2d726 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -144,6 +144,9 @@ To ensure code coverage checks never fail in GitHub Actions CI or PR validation ``` - **Integration Test Class Naming & Handling**: Integration test classes (classes that rely on containers or Testcontainers) MUST start with `IT` and MUST NOT end with `Test` or `Test.java` (e.g., `ITAzureBlobStorageService.java`, `ITAzureBlobConcatenationService.java`). This ensures Maven Surefire skips them during `mvn test` and Maven Failsafe runs them during `mvn verify`. When a container runtime is unavailable, integration test classes must be cleanly skipped via `Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable())`. Do not duplicate unit tests or introduce unnecessary mocking inside integration test classes. +### 20. Explicit Top-Level Class Imports +- Always use top-level `import` statements at the top of Java files instead of writing fully qualified package class names inline in method signatures or method bodies (e.g. add `import me.desair.tus.server.util.Utils;` at the top of the file and call `Utils.interruptStream(...)` instead of writing `me.desair.tus.server.util.Utils.interruptStream(...)`). + ## IETF Resumable Uploads for HTTP (RUFH) Spec Maintenance & Update Playbook ### 1. Spec Diff Review diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java index 2a4f92a2..253d4744 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java @@ -23,6 +23,7 @@ import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.util.InterruptibleInputStream; +import me.desair.tus.server.util.Utils; import org.apache.commons.lang3.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -222,7 +223,7 @@ private void interruptLocalStream(String idStr) { InterruptibleInputStream stream = streamRef.get(); if (stream != null) { log.info("Interrupting JVM-local stream for upload ID {}", idStr); - stream.interrupt(); + Utils.interruptStream(stream); } } } diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java b/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java index 96e12d15..4bbf6d48 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java @@ -19,6 +19,7 @@ import me.desair.tus.server.upload.UploadLock; import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.util.InterruptibleInputStream; +import me.desair.tus.server.util.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,16 +87,16 @@ private void closeQuietly() { @Override public void close() throws IOException { + synchronized (watchdogLock) { + if (watchdogThread != null) { + watchdogThread.interrupt(); + watchdogThread = null; + } + } + activeLocks.clear(); if (!closed) { closed = true; deregisterShutdownHook(); - synchronized (watchdogLock) { - if (watchdogThread != null) { - watchdogThread.interrupt(); - watchdogThread = null; - } - } - activeLocks.clear(); } } @@ -206,11 +207,14 @@ public void requestLockRelease(String requestUri) { // 1. Release JVM-local lock if active WeakReference streamRef = activeLocks.get(idStr); if (streamRef != null) { - InterruptibleInputStream stream = streamRef.get(); - if (stream != null) { - stream.interrupt(); + try { + InterruptibleInputStream stream = streamRef.get(); + if (stream != null) { + Utils.interruptStream(stream); + } + } finally { + activeLocks.remove(idStr); } - activeLocks.remove(idStr); } // 2. Create the stop file to signal other replicas @@ -290,31 +294,32 @@ public void run() { } private void checkActiveLocks() { - // Check stop files for each active lock + // Check stop files for each active lock safely per entry for (Map.Entry> entry : activeLocks.entrySet()) { String idStr = entry.getKey(); - WeakReference ref = entry.getValue(); - InterruptibleInputStream stream = ref.get(); + try { + WeakReference ref = entry.getValue(); + InterruptibleInputStream stream = ref != null ? ref.get() : null; - if (stream == null) { + if (stream == null) { + activeLocks.remove(idStr); + continue; + } + + checkStopFileAndInterrupt(idStr, stream); + } catch (Throwable t) { + log.warn("Error checking active lock for ID " + idStr, t); activeLocks.remove(idStr); - continue; } - - checkStopFileAndInterrupt(idStr, stream); } } private void checkStopFileAndInterrupt(String idStr, InterruptibleInputStream stream) { Path stopFilePath = getStopPath(new UploadId(idStr)); if (stopFilePath != null && Files.exists(stopFilePath)) { - try { - log.info("Watchdog detected stop file for upload ID {}. Interrupting stream.", idStr); - stream.interrupt(); - } catch (Throwable t) { - log.warn("Error interrupting stream for ID " + idStr, t); - } + log.info("Watchdog detected stop file for upload ID {}. Interrupting stream.", idStr); + Utils.interruptStream(stream); activeLocks.remove(idStr); try { Files.deleteIfExists(stopFilePath); diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index 620aa58e..39892303 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -26,7 +26,6 @@ import me.desair.tus.server.upload.UploadLock; import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UuidUploadIdFactory; -import me.desair.tus.server.util.InterruptibleInputStream; import me.desair.tus.server.util.S3UploadLockJsonSerializer; import me.desair.tus.server.util.Utils; import org.slf4j.Logger; @@ -229,9 +228,9 @@ public void requestLockRelease(String requestUri) { } // Step 1: Interrupt local active payload byte stream if hosted on this node - InputStream activeStream = activeInputStreams.get(requestUri); + InputStream activeStream = activeInputStreams.remove(requestUri); if (activeStream != null) { - interruptStream(activeStream); + Utils.interruptStream(activeStream); } // Step 2: Write a .stop signal object to S3 to signal lock contention across remote nodes/pods @@ -351,14 +350,7 @@ private void checkStopSignalForEntry(String uri, InputStream inputStream) { } private void interruptStream(InputStream is) { - if (is instanceof InterruptibleInputStream) { - ((InterruptibleInputStream) is).interrupt(); - } else { - try { - is.close(); - } catch (Exception ignored) { - } - } + Utils.interruptStream(is); } private void deleteObjectQuietly(String key) { diff --git a/src/main/java/me/desair/tus/server/util/Utils.java b/src/main/java/me/desair/tus/server/util/Utils.java index b93a7b11..776fbd6f 100644 --- a/src/main/java/me/desair/tus/server/util/Utils.java +++ b/src/main/java/me/desair/tus/server/util/Utils.java @@ -443,4 +443,26 @@ public static void shutdownExecutor(ScheduledExecutorService executor) { } } } + + /** + * Safely interrupts an input stream if it is an instance of {@link InterruptibleInputStream}, or + * closes it quietly if it is a standard input stream. Any exceptions encountered during + * interruption or closing are caught and logged without propagating. + * + * @param inputStream The InputStream to interrupt or close + */ + public static void interruptStream(java.io.InputStream inputStream) { + if (inputStream == null) { + return; + } + try { + if (inputStream instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) inputStream).interrupt(); + } else { + inputStream.close(); + } + } catch (Throwable t) { + log.warn("Error interrupting input stream: {}", t.getMessage(), t); + } + } } diff --git a/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java index b74ea409..98e7d077 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java @@ -23,6 +23,7 @@ import me.desair.tus.server.util.InterruptibleInputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; +import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -79,10 +80,17 @@ public UploadId answer(InvocationOnMock invocation) throws Throwable { lockingService = new DiskLockingService(idFactory, storagePath.toString()); } + @After + public void tearDown() throws Exception { + if (lockingService != null) { + lockingService.close(); + } + } + @Test public void lockUploadByUri() throws Exception { - UploadLock uploadLock = - lockingService.lockUploadByUri("/upload/test/000003f1-a850-49de-af03-997272d834c9"); + String uploadIdStr = UUID.randomUUID().toString(); + UploadLock uploadLock = lockingService.lockUploadByUri("/upload/test/" + uploadIdStr); assertThat(uploadLock, not(nullValue())); @@ -91,23 +99,21 @@ public void lockUploadByUri() throws Exception { @Test public void isLockedTrue() throws Exception { - UploadLock uploadLock = - lockingService.lockUploadByUri("/upload/test/000003f1-a850-49de-af03-997272d834c9"); + String uploadIdStr = UUID.randomUUID().toString(); + UploadLock uploadLock = lockingService.lockUploadByUri("/upload/test/" + uploadIdStr); - assertThat( - lockingService.isLocked(new UploadId("000003f1-a850-49de-af03-997272d834c9")), is(true)); + assertThat(lockingService.isLocked(new UploadId(uploadIdStr)), is(true)); uploadLock.release(); } @Test public void isLockedFalse() throws Exception { - UploadLock uploadLock = - lockingService.lockUploadByUri("/upload/test/000003f1-a850-49de-af03-997272d834c9"); + String uploadIdStr = UUID.randomUUID().toString(); + UploadLock uploadLock = lockingService.lockUploadByUri("/upload/test/" + uploadIdStr); uploadLock.release(); - assertThat( - lockingService.isLocked(new UploadId("000003f1-a850-49de-af03-997272d834c9")), is(false)); + assertThat(lockingService.isLocked(new UploadId(uploadIdStr)), is(false)); } @Test @@ -115,8 +121,7 @@ public void lockUploadNotExists() throws Exception { reset(idFactory); when(idFactory.readUploadId(nullable(String.class))).thenReturn(null); - UploadLock uploadLock = - lockingService.lockUploadByUri("/upload/test/000003f1-a850-49de-af03-997272d834c9"); + UploadLock uploadLock = lockingService.lockUploadByUri("/upload/test/" + UUID.randomUUID()); assertThat(uploadLock, nullValue()); } @@ -125,7 +130,7 @@ public void lockUploadNotExists() throws Exception { public void cleanupStaleLocks() throws Exception { Path locksPath = storagePath.resolve("locks"); - String activeLock = "000003f1-a850-49de-af03-997272d834c9"; + String activeLock = UUID.randomUUID().toString(); UploadLock uploadLock = lockingService.lockUploadByUri("/upload/test/" + activeLock); assertThat(uploadLock, not(nullValue())); @@ -156,7 +161,8 @@ public void cleanupStaleLocks() throws Exception { @Test public void testRegisterAndRequestLockReleaseLocal() throws Exception { - String uri = "/upload/test/000003f1-a850-49de-af03-997272d834c9"; + String uploadIdStr = UUID.randomUUID().toString(); + String uri = "/upload/test/" + uploadIdStr; byte[] data = new byte[] {1, 2, 3}; ByteArrayInputStream bis = new ByteArrayInputStream(data); InterruptibleInputStream iis = new InterruptibleInputStream(bis); @@ -168,15 +174,15 @@ public void testRegisterAndRequestLockReleaseLocal() throws Exception { assertTrue(iis.isInterrupted()); // Stop file should also be created - Path stopFilePath = - storagePath.resolve("locks").resolve("000003f1-a850-49de-af03-997272d834c9.stop"); + Path stopFilePath = storagePath.resolve("locks").resolve(uploadIdStr + ".stop"); assertTrue(Files.exists(stopFilePath)); Files.deleteIfExists(stopFilePath); } @Test public void testWatchdogInterruptsStreamOnStopFile() throws Exception { - String uri = "/upload/test/000003f1-a850-49de-af03-997272d834c9"; + String uploadIdStr = UUID.randomUUID().toString(); + String uri = "/upload/test/" + uploadIdStr; byte[] data = new byte[] {1, 2, 3}; ByteArrayInputStream bis = new ByteArrayInputStream(data); InterruptibleInputStream iis = new InterruptibleInputStream(bis); @@ -185,15 +191,14 @@ public void testWatchdogInterruptsStreamOnStopFile() throws Exception { assertFalse(iis.isInterrupted()); // Manually create the stop file (simulating cross-replica signaling) - Path stopFilePath = - storagePath.resolve("locks").resolve("000003f1-a850-49de-af03-997272d834c9.stop"); + Path stopFilePath = storagePath.resolve("locks").resolve(uploadIdStr + ".stop"); Files.createDirectories(stopFilePath.getParent()); Files.write(stopFilePath, new byte[0]); - // Wait for watchdog to poll (polls every 1000ms, wait up to 2.5s) + // Wait for watchdog to poll (polls every 1000ms, wait up to 5s) long start = System.currentTimeMillis(); - while (!iis.isInterrupted() && System.currentTimeMillis() - start < 2500L) { - Thread.sleep(100L); + while (!iis.isInterrupted() && System.currentTimeMillis() - start < 5000L) { + Thread.sleep(50L); } assertTrue("Watchdog should have interrupted the stream", iis.isInterrupted()); @@ -202,7 +207,8 @@ public void testWatchdogInterruptsStreamOnStopFile() throws Exception { @Test public void testWatchdogTerminatesWhenEmpty() throws Exception { - String uri = "/upload/test/000003f1-a850-49de-af03-997272d834c9"; + String uploadIdStr = UUID.randomUUID().toString(); + String uri = "/upload/test/" + uploadIdStr; byte[] data = new byte[] {1, 2, 3}; ByteArrayInputStream bis = new ByteArrayInputStream(data); InterruptibleInputStream iis = new InterruptibleInputStream(bis); @@ -218,14 +224,13 @@ public void testWatchdogTerminatesWhenEmpty() throws Exception { // Watchdog should stop (since loop exits after activeLocks is empty) long start = System.currentTimeMillis(); - while (watchdog.isAlive() && System.currentTimeMillis() - start < 2500L) { - Thread.sleep(100L); + while (watchdog.isAlive() && System.currentTimeMillis() - start < 5000L) { + Thread.sleep(50L); } assertFalse("Watchdog thread should have terminated", watchdog.isAlive()); // Clean up stop file - Path stopFilePath = - storagePath.resolve("locks").resolve("000003f1-a850-49de-af03-997272d834c9.stop"); + Path stopFilePath = storagePath.resolve("locks").resolve(uploadIdStr + ".stop"); Files.deleteIfExists(stopFilePath); } @@ -233,17 +238,16 @@ public void testWatchdogTerminatesWhenEmpty() throws Exception { public void testDefaultConstructor() throws Exception { DiskLockingService defaultService = new DiskLockingService(storagePath.toString()); defaultService.setIdFactory(idFactory); - UploadLock lock = - defaultService.lockUploadByUri("/upload/test/000003f1-a850-49de-af03-997272d834c9"); + UploadLock lock = defaultService.lockUploadByUri("/upload/test/" + UUID.randomUUID()); assertThat(lock, not(nullValue())); lock.close(); } @Test public void testRequestLockReleaseIOException() throws Exception { - String uri = "/upload/test/000003f1-a850-49de-af03-997272d834c9"; - Path stopFilePath = - storagePath.resolve("locks").resolve("000003f1-a850-49de-af03-997272d834c9.stop"); + String uploadIdStr = UUID.randomUUID().toString(); + String uri = "/upload/test/" + uploadIdStr; + Path stopFilePath = storagePath.resolve("locks").resolve(uploadIdStr + ".stop"); Files.createDirectories(stopFilePath); try { @@ -322,7 +326,8 @@ public void testCleanupStaleLocksWithStaleLockAndStopFile() throws Exception { @Test public void testWatchdogRobustnessOnInterruptException() throws Exception { - String uri = "/upload/test/000003f1-a850-49de-af03-997272d834c9"; + String uploadIdStr = UUID.randomUUID().toString(); + String uri = "/upload/test/" + uploadIdStr; InterruptibleInputStream faultyStream = new InterruptibleInputStream(new ByteArrayInputStream(new byte[0])) { @@ -334,14 +339,13 @@ public void interrupt() { lockingService.registerInputStream(uri, faultyStream); - Path stopFilePath = - storagePath.resolve("locks").resolve("000003f1-a850-49de-af03-997272d834c9.stop"); + Path stopFilePath = storagePath.resolve("locks").resolve(uploadIdStr + ".stop"); Files.createDirectories(stopFilePath.getParent()); Files.write(stopFilePath, new byte[0]); long start = System.currentTimeMillis(); - while (Files.exists(stopFilePath) && System.currentTimeMillis() - start < 2500L) { - Thread.sleep(100L); + while (Files.exists(stopFilePath) && System.currentTimeMillis() - start < 5000L) { + Thread.sleep(50L); } Files.deleteIfExists(stopFilePath); @@ -353,15 +357,16 @@ public void testRequestLockReleaseCreatesParentDirectory() throws Exception { Path nestedStorage = tempDir.resolve("nested").resolve("sub"); DiskLockingService service = new DiskLockingService(idFactory, nestedStorage.toString()); - UploadId id = new UploadId("000003f1-a850-49de-af03-997272d834c9"); + String uploadIdStr = UUID.randomUUID().toString(); + UploadId id = new UploadId(uploadIdStr); java.lang.reflect.Field urlSafeField = UploadId.class.getDeclaredField("urlSafeValue"); urlSafeField.setAccessible(true); - urlSafeField.set(id, "subdir/000003f1-a850-49de-af03-997272d834c9"); + urlSafeField.set(id, "subdir/" + uploadIdStr); reset(idFactory); when(idFactory.readUploadId(org.mockito.Mockito.anyString())).thenReturn(id); - String uri = "/upload/test/subdir/000003f1-a850-49de-af03-997272d834c9"; + String uri = "/upload/test/subdir/" + uploadIdStr; service.requestLockRelease(uri); Path stopFilePath = @@ -369,16 +374,18 @@ public void testRequestLockReleaseCreatesParentDirectory() throws Exception { .resolve("locks") .resolve("subdir") .resolve("subdir") - .resolve("000003f1-a850-49de-af03-997272d834c9.stop"); + .resolve(uploadIdStr + ".stop"); assertTrue(Files.exists(stopFilePath)); Files.deleteIfExists(stopFilePath); + service.close(); FileUtils.deleteDirectory(tempDir.toFile()); } @Test public void testRequestLockReleaseWithGCedStream() throws Exception { - String uri = "/upload/test/000003f1-a850-49de-af03-997272d834c9"; + String uploadIdStr = UUID.randomUUID().toString(); + String uri = "/upload/test/" + uploadIdStr; InterruptibleInputStream iis = new InterruptibleInputStream(new ByteArrayInputStream(new byte[0])); lockingService.registerInputStream(uri, iis); @@ -392,17 +399,17 @@ public void testRequestLockReleaseWithGCedStream() throws Exception { String, java.lang.ref.WeakReference>) field.get(null); - java.lang.ref.WeakReference ref = - map.get("000003f1-a850-49de-af03-997272d834c9"); + java.lang.ref.WeakReference ref = map.get(uploadIdStr); ref.clear(); lockingService.requestLockRelease(uri); - assertFalse(map.containsKey("000003f1-a850-49de-af03-997272d834c9")); + assertFalse(map.containsKey(uploadIdStr)); } @Test public void testRegisteredLockGetUploadUri() throws Exception { - String uri = "/upload/test/000003f1-a850-49de-af03-997272d834c9"; + String uploadIdStr = UUID.randomUUID().toString(); + String uri = "/upload/test/" + uploadIdStr; UploadLock lock = lockingService.lockUploadByUri(uri); org.junit.Assert.assertNotNull(lock); assertThat(lock.getUploadUri(), is(uri)); @@ -411,7 +418,8 @@ public void testRegisteredLockGetUploadUri() throws Exception { @Test public void testWatchdogRemovesClearedWeakReference() throws Exception { - String uri = "/upload/test/000003f1-a850-49de-af03-997272d834c9"; + String uploadIdStr = UUID.randomUUID().toString(); + String uri = "/upload/test/" + uploadIdStr; InterruptibleInputStream iis = new InterruptibleInputStream(new ByteArrayInputStream(new byte[0])); lockingService.registerInputStream(uri, iis); @@ -425,25 +433,24 @@ public void testWatchdogRemovesClearedWeakReference() throws Exception { String, java.lang.ref.WeakReference>) field.get(null); - java.lang.ref.WeakReference ref = - map.get("000003f1-a850-49de-af03-997272d834c9"); + java.lang.ref.WeakReference ref = map.get(uploadIdStr); org.junit.Assert.assertNotNull(ref); ref.clear(); long start = System.currentTimeMillis(); - while (map.containsKey("000003f1-a850-49de-af03-997272d834c9") - && System.currentTimeMillis() - start < 2500L) { - Thread.sleep(100L); + while (map.containsKey(uploadIdStr) && System.currentTimeMillis() - start < 5000L) { + Thread.sleep(50L); } assertFalse( "Watchdog should have removed the cleared weak reference from activeLocks", - map.containsKey("000003f1-a850-49de-af03-997272d834c9")); + map.containsKey(uploadIdStr)); } @Test public void testWatchdogInterrupted() throws Exception { - String uri = "/upload/test/000003f1-a850-49de-af03-997272d834c9"; + String uploadIdStr = UUID.randomUUID().toString(); + String uri = "/upload/test/" + uploadIdStr; InterruptibleInputStream iis = new InterruptibleInputStream(new ByteArrayInputStream(new byte[0])); lockingService.registerInputStream(uri, iis); @@ -455,8 +462,8 @@ public void testWatchdogInterrupted() throws Exception { watchdog.interrupt(); long start = System.currentTimeMillis(); - while (watchdog.isAlive() && System.currentTimeMillis() - start < 2500L) { - Thread.sleep(100L); + while (watchdog.isAlive() && System.currentTimeMillis() - start < 5000L) { + Thread.sleep(50L); } assertFalse("Watchdog thread should have terminated on interruption", watchdog.isAlive()); @@ -478,7 +485,8 @@ public void testWatchdogUnexpectedException() throws Exception { String, java.lang.ref.WeakReference>) field.get(null); - String uri = "/upload/test/000003f1-a850-49de-af03-997272d834c9"; + String uploadIdStr = UUID.randomUUID().toString(); + String uri = "/upload/test/" + uploadIdStr; InterruptibleInputStream iis = new InterruptibleInputStream(new ByteArrayInputStream(new byte[0])); lockingService.registerInputStream(uri, iis); @@ -489,11 +497,11 @@ public void testWatchdogUnexpectedException() throws Exception { java.lang.ref.WeakReference mockRef = org.mockito.Mockito.mock(java.lang.ref.WeakReference.class); when(mockRef.get()).thenThrow(new RuntimeException("Simulated exception")); - map.put("trigger-error", mockRef); + map.put("trigger-error-" + UUID.randomUUID(), mockRef); long start = System.currentTimeMillis(); - while (watchdog.isAlive() && System.currentTimeMillis() - start < 2500L) { - Thread.sleep(100L); + while (watchdog.isAlive() && System.currentTimeMillis() - start < 5000L) { + Thread.sleep(50L); } map.clear(); @@ -501,12 +509,12 @@ public void testWatchdogUnexpectedException() throws Exception { @Test public void testRegisteredLockDeleteStopFileIOException() throws Exception { - String uri = "/upload/test/000003f1-a850-49de-af03-997272d834c9"; + String uploadIdStr = UUID.randomUUID().toString(); + String uri = "/upload/test/" + uploadIdStr; UploadLock lock = lockingService.lockUploadByUri(uri); org.junit.Assert.assertNotNull(lock); - Path stopFilePath = - storagePath.resolve("locks").resolve("000003f1-a850-49de-af03-997272d834c9.stop"); + Path stopFilePath = storagePath.resolve("locks").resolve(uploadIdStr + ".stop"); Files.createDirectories(stopFilePath); Files.createFile(stopFilePath.resolve("dummy")); @@ -558,12 +566,13 @@ public void cleanupStaleLocksWhenStorageDirectoryNotExists() throws Exception { assertTrue(Files.exists(nonExistentPath.resolve("locks"))); // Cleanup + newLockingService.close(); FileUtils.deleteDirectory(nonExistentPath.toFile()); } @Test public void testRequestLockReleaseNullLockPath() throws Exception { - String uri = "/upload/test/000003f1-a850-49de-af03-997272d834c9"; + String uri = "/upload/test/" + UUID.randomUUID(); // Mock ID factory to return an ID that will result in a null lock path // We can just return a null UploadId to get null from getPathInStorageDirectory diff --git a/src/test/java/me/desair/tus/server/util/UtilsTest.java b/src/test/java/me/desair/tus/server/util/UtilsTest.java index 6f7f6131..5dbce7ad 100644 --- a/src/test/java/me/desair/tus/server/util/UtilsTest.java +++ b/src/test/java/me/desair/tus/server/util/UtilsTest.java @@ -572,6 +572,37 @@ public void testShutdownExecutorWithException() { Utils.shutdownExecutor(mockExecutor); } + @Test + public void testInterruptStreamNull() { + Utils.interruptStream(null); + } + + @Test + public void testInterruptStreamStandardInputStream() throws Exception { + java.io.ByteArrayInputStream bis = new java.io.ByteArrayInputStream(new byte[0]); + Utils.interruptStream(bis); + } + + @Test + public void testInterruptStreamInterruptibleInputStream() { + InterruptibleInputStream iis = + new InterruptibleInputStream(new java.io.ByteArrayInputStream(new byte[0])); + Utils.interruptStream(iis); + assertThat(iis.isInterrupted(), is(true)); + } + + @Test + public void testInterruptStreamWithException() { + InterruptibleInputStream faultyStream = + new InterruptibleInputStream(new java.io.ByteArrayInputStream(new byte[0])) { + @Override + public void interrupt() { + throw new RuntimeException("Error during interrupt"); + } + }; + Utils.interruptStream(faultyStream); + } + /** Simple serializable class for testing. */ public static class TestSerializable implements Serializable { private static final long serialVersionUID = 1L; From 66938ec9c9039b92fd26a3275ff05f969de08b5a Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Wed, 12 Aug 2026 12:46:01 +0200 Subject: [PATCH 10/14] fix(ci): restore jacoco-it.exec report generation by removing surefireArgLine and failsafeArgLine from properties --- pom.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pom.xml b/pom.xml index 8c26f9bd..97e2b9df 100644 --- a/pom.xml +++ b/pom.xml @@ -23,8 +23,6 @@ - - From b4f46f8987cc3a7f9ad753ec4ef7ddfdead7a949 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Wed, 12 Aug 2026 13:01:06 +0200 Subject: [PATCH 11/14] test: add out-of-order concatenation integration test and committed block IDs test --- .../AbstractITTusFileUploadService.java | 99 +++++++++++++++++++ .../azure/ITAzureBlobLockingService.java | 20 ++++ .../azure/ITAzureBlobStorageService.java | 34 +++++++ .../upload/s3/S3LockingServiceTest.java | 24 +++++ 4 files changed, 177 insertions(+) diff --git a/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java b/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java index c558fb36..77afd56b 100644 --- a/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java +++ b/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java @@ -1054,6 +1054,105 @@ public void testConcatenationCompleted() throws Exception { } } + @Test + public void testConcatenationOutOfOrderCompleted() throws Exception { + String part1 = "This is the first part of my test upload "; + String part2 = "and this is the second part."; + + // 1. Create the SECOND upload part FIRST + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.CONTENT_LENGTH, 0); + servletRequest.addHeader(HttpHeader.UPLOAD_LENGTH, "28"); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + servletRequest.addHeader(HttpHeader.UPLOAD_CONCAT, "partial"); + servletRequest.addHeader(HttpHeader.UPLOAD_METADATA, "filename cGFydDIudHh0"); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_CREATED); + String location2 = + UPLOAD_URI + + StringUtils.substringAfter( + servletResponse.getHeader(HttpHeader.LOCATION), UPLOAD_URI); + + // Upload part 2 bytes first + reset(); + servletRequest.setMethod("PATCH"); + servletRequest.setRequestURI(location2); + servletRequest.addHeader(HttpHeader.CONTENT_TYPE, "application/offset+octet-stream"); + servletRequest.addHeader(HttpHeader.CONTENT_LENGTH, "28"); + servletRequest.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + servletRequest.setContent(part2.getBytes()); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_NO_CONTENT); + + // 2. Create the FIRST upload part SECOND + reset(); + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.CONTENT_LENGTH, 0); + servletRequest.addHeader(HttpHeader.UPLOAD_LENGTH, "41"); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + servletRequest.addHeader(HttpHeader.UPLOAD_CONCAT, "partial"); + servletRequest.addHeader(HttpHeader.UPLOAD_METADATA, "filename cGFydDEudHh0"); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_CREATED); + String location1 = + UPLOAD_URI + + StringUtils.substringAfter( + servletResponse.getHeader(HttpHeader.LOCATION), UPLOAD_URI); + + // Upload part 1 bytes second + reset(); + servletRequest.setMethod("PATCH"); + servletRequest.setRequestURI(location1); + servletRequest.addHeader(HttpHeader.CONTENT_TYPE, "application/offset+octet-stream"); + servletRequest.addHeader(HttpHeader.CONTENT_LENGTH, "41"); + servletRequest.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + servletRequest.setContent(part1.getBytes()); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_NO_CONTENT); + + // 3. Create final concatenated upload referencing part 1 then part 2 in correct order + reset(); + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.CONTENT_LENGTH, 0); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + servletRequest.addHeader(HttpHeader.UPLOAD_CONCAT, "final ; " + location1 + " " + location2); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_CREATED); + String finalLocation = + UPLOAD_URI + + StringUtils.substringAfter( + servletResponse.getHeader(HttpHeader.LOCATION), UPLOAD_URI); + + // Download and verify content is in the correct concatenated order (part1 + part2) + reset(); + servletRequest.setMethod("GET"); + servletRequest.setRequestURI(finalLocation); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_OK); + assertThat( + servletResponse.getContentAsString(), + is("This is the first part of my test upload and this is the second part.")); + + try (InputStream uploadedBytes = + tusFileUploadService.getUploadedBytes(finalLocation, OWNER_KEY)) { + assertThat( + IOUtils.toString(uploadedBytes, StandardCharsets.UTF_8), + is("This is the first part of my test upload and this is the second part.")); + } + } + @Test public void testConcatenationUnfinished() throws Exception { String part1 = "When sending this part, the final upload was already created. "; diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java index a54d0a94..f646e5a3 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java @@ -207,4 +207,24 @@ public void lockUploadByUriShouldThrowIOExceptionOnStorageException() throws Exc service.lockUploadByUri("/test/upload/12345"); } + + @Test + public void testCreateAndDeleteStopSignalBlobExceptionHandling() { + com.azure.storage.blob.BlobServiceClient serviceClient = containerClient.getServiceClient(); + BlobContainerClient invalidContainer = + serviceClient.getBlobContainerClient("invalid-container-" + System.currentTimeMillis()); + + AzureBlobLockingService invalidLocking = new AzureBlobLockingService(invalidContainer); + TimeBasedUploadIdFactory idFactory = new TimeBasedUploadIdFactory(); + idFactory.setUploadUri("/test/upload"); + invalidLocking.setIdFactory(idFactory); + + ByteArrayInputStream bais = new ByteArrayInputStream("data".getBytes()); + InterruptibleInputStream stream = new InterruptibleInputStream(bais); + invalidLocking.registerInputStream("/test/upload/12345", stream); + + // Requesting lock release triggers createStopSignalBlob & deleteStopSignalBlob on invalid + // container + invalidLocking.requestLockRelease("/test/upload/12345"); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java index d6555ea3..b3ea6077 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java @@ -493,4 +493,38 @@ public void testDeduplicationOnAzurite() throws Exception { assertNull( storage.getUploadInfoByChecksum("5d41402abc4b2a76b9719d911017c592", ChecksumAlgorithm.MD5)); } + + @Test + public void appendConsecutiveCommittedBlocksShouldRetrieveCommittedBlockIds() throws Exception { + storageService.setPreferredBlockSize(4L * 1024 * 1024); // 4MB block size + + byte[] block1 = new byte[4 * 1024 * 1024]; + java.util.Arrays.fill(block1, (byte) 'X'); + + byte[] block2 = new byte[4 * 1024 * 1024]; + java.util.Arrays.fill(block2, (byte) 'Y'); + + UploadInfo info = new UploadInfo(); + info.setLength(10L * 1024 * 1024); + UploadInfo created = storageService.create(info, "owner1"); + + // 1st append commits block 1 (4MB) + storageService.append(created, new ByteArrayInputStream(block1)); + assertEquals(Long.valueOf(4L * 1024 * 1024), created.getOffset()); + + // 2nd append calls getCommittedBlockIds(blockBlobClient) and commits block 2 (4MB) + storageService.append(created, new ByteArrayInputStream(block2)); + assertEquals(Long.valueOf(8L * 1024 * 1024), created.getOffset()); + + // 3rd append calls getCommittedBlockIds(blockBlobClient) which retrieves 2 committed block IDs + // from Azurite + storageService.append(created, new ByteArrayInputStream("extra".getBytes())); + assertEquals(Long.valueOf(8L * 1024 * 1024 + 5), created.getOffset()); + + try (InputStream is = storageService.getUploadedBytes(created.getId())) { + assertNotNull(is); + byte[] readBytes = org.apache.commons.io.IOUtils.toByteArray(is); + assertEquals(8 * 1024 * 1024 + 5, readBytes.length); + } + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index aed5d1e1..81fe8741 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -274,4 +274,28 @@ public void testSanitizePrefixNullOrEmpty() throws Exception { public void testClose() throws Exception { lockingService.close(); } + + @Test + public void testCheckStopSignalForEntryExceptionAndNullId() throws Exception { + lockingService.setIdFactory(new me.desair.tus.server.upload.TimeBasedUploadIdFactory()); + io.minio.MinioClient mockClient = Mockito.mock(io.minio.MinioClient.class); + Mockito.when(mockClient.statObject(Mockito.any(io.minio.StatObjectArgs.class))) + .thenThrow(new RuntimeException("General S3 Exception")); + Mockito.doThrow(new RuntimeException("Remove object failed")) + .when(mockClient) + .removeObject(Mockito.any(io.minio.RemoveObjectArgs.class)); + + S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = + new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); + idFactory.setUploadUri("/files/upload"); + service.setIdFactory(idFactory); + + ByteArrayInputStream bais = new ByteArrayInputStream("test".getBytes()); + InterruptibleInputStream stream = new InterruptibleInputStream(bais); + + service.registerInputStream("/files/upload/12345", stream); + // Triggers checkStopSignalForEntry & deleteObjectQuietly which catch RuntimeException + service.requestLockRelease("/files/upload/12345"); + } } From 7d5f7964902deb4aa8c1a12c7c7fbd3585f45930 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Wed, 12 Aug 2026 22:51:24 +0200 Subject: [PATCH 12/14] test: add unit tests for UploadLockingService default methods, S3 checkStopSignal happy path, and TusFileUploadService single-arg methods --- .../tus/server/TusFileUploadServiceTest.java | 39 +++++++ .../upload/UploadLockingServiceTest.java | 22 ++-- .../upload/s3/S3LockingServiceTest.java | 108 ++++++++++++++++++ 3 files changed, 160 insertions(+), 9 deletions(-) diff --git a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java index 8d1cb1ce..a1e9210e 100644 --- a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java +++ b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java @@ -533,4 +533,43 @@ public void testClose() throws Exception { verify(mockLockingService).close(); } + + @Test + public void testGetUploadInfoSingleArg() throws Exception { + UploadLockingService mockLockingService = mock(UploadLockingService.class); + UploadStorageService mockStorageService = mock(UploadStorageService.class); + UploadLock mockLock = mock(UploadLock.class); + UploadInfo mockInfo = new UploadInfo(); + + when(mockLockingService.lockUploadByUri(anyString())).thenReturn(mockLock); + when(mockStorageService.getUploadInfo("/files/123", null)).thenReturn(mockInfo); + + TusFileUploadService service = + new TusFileUploadService() + .withUploadLockingService(mockLockingService) + .withUploadStorageService(mockStorageService); + + UploadInfo result = service.getUploadInfo("/files/123"); + assertNotNull(result); + verify(mockStorageService).getUploadInfo("/files/123", null); + } + + @Test + public void testDeleteUploadSingleArg() throws Exception { + UploadLockingService mockLockingService = mock(UploadLockingService.class); + UploadStorageService mockStorageService = mock(UploadStorageService.class); + UploadLock mockLock = mock(UploadLock.class); + UploadInfo mockInfo = new UploadInfo(); + + when(mockLockingService.lockUploadByUri(anyString())).thenReturn(mockLock); + when(mockStorageService.getUploadInfo("/files/123", null)).thenReturn(mockInfo); + + TusFileUploadService service = + new TusFileUploadService() + .withUploadLockingService(mockLockingService) + .withUploadStorageService(mockStorageService); + + service.deleteUpload("/files/123"); + verify(mockStorageService).terminateUpload(mockInfo); + } } diff --git a/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java index d63cf590..10b7d436 100644 --- a/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java @@ -1,23 +1,25 @@ package me.desair.tus.server.upload; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertFalse; import java.io.ByteArrayInputStream; +import java.io.IOException; +import me.desair.tus.server.exception.TusException; import org.junit.Test; public class UploadLockingServiceTest { @Test - public void testDefaultMethods() { - UploadLockingService service = + public void testDefaultMethods() throws Exception { + UploadLockingService dummyService = new UploadLockingService() { @Override - public UploadLock lockUploadByUri(String requestUri) { + public UploadLock lockUploadByUri(String requestUri) throws TusException, IOException { return null; } @Override - public void cleanupStaleLocks() {} + public void cleanupStaleLocks() throws IOException {} @Override public boolean isLocked(UploadId id) { @@ -28,9 +30,11 @@ public boolean isLocked(UploadId id) { public void setIdFactory(UploadIdFactory idFactory) {} }; - // Verify default methods do not throw exceptions and act as no-ops - service.registerInputStream("/files/test", new ByteArrayInputStream(new byte[0])); - service.requestLockRelease("/files/test"); - assertNotNull(service); + // Test default methods for coverage + dummyService.registerInputStream("/test/upload/123", new ByteArrayInputStream(new byte[0])); + dummyService.requestLockRelease("/test/upload/123"); + dummyService.close(); + + assertFalse(dummyService.isLocked(new UploadId("123"))); } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index 81fe8741..7cb6e8a3 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -298,4 +298,112 @@ public void testCheckStopSignalForEntryExceptionAndNullId() throws Exception { // Triggers checkStopSignalForEntry & deleteObjectQuietly which catch RuntimeException service.requestLockRelease("/files/upload/12345"); } + + @Test + public void testCheckStopSignalForEntryHappyPath() throws Exception { + io.minio.MinioClient mockClient = Mockito.mock(io.minio.MinioClient.class); + io.minio.StatObjectResponse mockStat = Mockito.mock(io.minio.StatObjectResponse.class); + Mockito.when(mockClient.statObject(Mockito.any(io.minio.StatObjectArgs.class))) + .thenReturn(mockStat); + + S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = + new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); + idFactory.setUploadUri("/files/upload"); + service.setIdFactory(idFactory); + + ByteArrayInputStream bais = new ByteArrayInputStream("test".getBytes()); + InterruptibleInputStream stream = new InterruptibleInputStream(bais); + + service.registerInputStream("/files/upload/12345", stream); + service.requestLockRelease("/files/upload/12345"); + + org.junit.Assert.assertTrue(stream.isInterrupted()); + } + + @Test + public void testCleanupStaleLocksWithExpiredAndNonExpiredLocks() throws Exception { + MinioClient mockClient = Mockito.mock(MinioClient.class); + Item expiredItem = Mockito.mock(Item.class); + Mockito.when(expiredItem.objectName()).thenReturn("locks/expired.lock"); + + Item validItem = Mockito.mock(Item.class); + Mockito.when(validItem.objectName()).thenReturn("locks/valid.lock"); + + Result res1 = new Result(expiredItem); + Result res2 = new Result(validItem); + + Mockito.when(mockClient.listObjects(Mockito.any(ListObjectsArgs.class))) + .thenReturn(java.util.Arrays.asList(res1, res2)); + + S3UploadLock expiredLock = + new S3UploadLock( + "holder1", + "/files/upload/1", + "test-bucket", + "locks/expired.lock", + "locks/expired.stop", + 30000L, + 1000L); + S3UploadLock validLock = + new S3UploadLock( + "holder2", + "/files/upload/2", + "test-bucket", + "locks/valid.lock", + "locks/valid.stop", + 30000L, + System.currentTimeMillis() + 1000000L); + + GetObjectResponse expiredStream = + new GetObjectResponse( + null, + "test-bucket", + "us-east-1", + "locks/expired.lock", + new ByteArrayInputStream( + me.desair.tus.server.util.S3UploadLockJsonSerializer.serialize(expiredLock) + .getBytes(StandardCharsets.UTF_8))); + GetObjectResponse validStream = + new GetObjectResponse( + null, + "test-bucket", + "us-east-1", + "locks/valid.lock", + new ByteArrayInputStream( + me.desair.tus.server.util.S3UploadLockJsonSerializer.serialize(validLock) + .getBytes(StandardCharsets.UTF_8))); + + Mockito.when(mockClient.getObject(Mockito.any(GetObjectArgs.class))) + .thenReturn(expiredStream) + .thenReturn(validStream); + + S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + service.cleanupStaleLocks(); + + // Expired lock object is deleted + Mockito.verify(mockClient).removeObject(Mockito.any(io.minio.RemoveObjectArgs.class)); + } + + @Test + public void testRegisterInputStreamNullChecks() { + lockingService.registerInputStream(null, Mockito.mock(InputStream.class)); + lockingService.registerInputStream("/files/upload/123", null); + } + + @Test + public void testWriteStopSignalException() throws Exception { + MinioClient mockClient = Mockito.mock(MinioClient.class); + Mockito.when(mockClient.putObject(Mockito.any(PutObjectArgs.class))) + .thenThrow(new RuntimeException("S3 Put error")); + + S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = + new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); + idFactory.setUploadUri("/files/upload"); + service.setIdFactory(idFactory); + + // requestLockRelease calls writeStopSignal which catches RuntimeException + service.requestLockRelease("/files/upload/12345"); + } } From 4b51a8dd8f115cfddc7c9468441b6f22db0e5225 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Wed, 12 Aug 2026 23:04:19 +0200 Subject: [PATCH 13/14] test: assert statObject invocation during background watchdog polling in S3LockingServiceTest --- .../upload/s3/S3LockingServiceTest.java | 52 ++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index 7cb6e8a3..0435f3fb 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -301,12 +301,12 @@ public void testCheckStopSignalForEntryExceptionAndNullId() throws Exception { @Test public void testCheckStopSignalForEntryHappyPath() throws Exception { - io.minio.MinioClient mockClient = Mockito.mock(io.minio.MinioClient.class); + MinioClient mockClient = Mockito.mock(MinioClient.class); io.minio.StatObjectResponse mockStat = Mockito.mock(io.minio.StatObjectResponse.class); - Mockito.when(mockClient.statObject(Mockito.any(io.minio.StatObjectArgs.class))) - .thenReturn(mockStat); + Mockito.when(mockClient.statObject(Mockito.any(StatObjectArgs.class))).thenReturn(mockStat); - S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + S3LockingService service = + new S3LockingService(mockClient, "test-bucket", "locks", 30000L, 50L); me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); idFactory.setUploadUri("/files/upload"); @@ -316,9 +316,49 @@ public void testCheckStopSignalForEntryHappyPath() throws Exception { InterruptibleInputStream stream = new InterruptibleInputStream(bais); service.registerInputStream("/files/upload/12345", stream); - service.requestLockRelease("/files/upload/12345"); - org.junit.Assert.assertTrue(stream.isInterrupted()); + // Wait for background watchdog thread to execute checkStopSignals() + long deadline = System.currentTimeMillis() + 2000L; + while (!stream.isInterrupted() && System.currentTimeMillis() < deadline) { + Thread.sleep(50L); + } + + assertTrue(stream.isInterrupted()); + Mockito.verify(mockClient, Mockito.atLeastOnce()).statObject(Mockito.any(StatObjectArgs.class)); + + service.close(); + } + + @Test + public void testCheckStopSignalForEntryNoSuchKey() throws Exception { + MinioClient mockClient = Mockito.mock(MinioClient.class); + ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(errorResponse.code()).thenReturn("NoSuchKey"); + ErrorResponseException noSuchKeyException = + new ErrorResponseException(errorResponse, null, "NoSuchKey"); + + Mockito.when(mockClient.statObject(Mockito.any(StatObjectArgs.class))) + .thenThrow(noSuchKeyException); + + S3LockingService service = + new S3LockingService(mockClient, "test-bucket", "locks", 30000L, 50L); + me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = + new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); + idFactory.setUploadUri("/files/upload"); + service.setIdFactory(idFactory); + + ByteArrayInputStream bais = new ByteArrayInputStream("test".getBytes()); + InterruptibleInputStream stream = new InterruptibleInputStream(bais); + + service.registerInputStream("/files/upload/12345", stream); + + // Wait for background watchdog thread to run checkStopSignals() + Thread.sleep(200L); + + Mockito.verify(mockClient, Mockito.atLeastOnce()).statObject(Mockito.any(StatObjectArgs.class)); + assertFalse(stream.isInterrupted()); + + service.close(); } @Test From f45ae64d711fbd1c3896885a1628354966e660f6 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Wed, 12 Aug 2026 23:12:20 +0200 Subject: [PATCH 14/14] test: add tests for empty part IDs and in-progress stream fallback in ITAzureBlobConcatenationService --- .../ITAzureBlobConcatenationService.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobConcatenationService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobConcatenationService.java index 32fb8687..c774db75 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobConcatenationService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobConcatenationService.java @@ -241,4 +241,32 @@ public void constructorPrefixSanitizationVariants() { assertNotNull(service2); assertNotNull(service3); } + + @Test + public void getConcatenatedBytesShouldReturnEmptyStreamForInProgressUploadWithIncompletePartials() + throws Exception { + UploadInfo part1Info = new UploadInfo(); + part1Info.setLength(10L); // length 10, offset 0 (incomplete) + UploadInfo part1 = storageService.create(part1Info, null); + + UploadInfo finalInfo = new UploadInfo(); + finalInfo.setConcatenationPartIds(Arrays.asList("/test/upload/" + part1.getId())); + finalInfo.setUploadType(UploadType.CONCATENATED); + UploadInfo createdFinal = storageService.create(finalInfo, null); + + InputStream is = concatenationService.getConcatenatedBytes(createdFinal); + assertNotNull(is); + assertEquals(0, is.available()); + } + + @Test + public void mergeWithEmptyPartIdsListShouldDoNothing() throws Exception { + UploadInfo finalInfo = new UploadInfo(); + finalInfo.setConcatenationPartIds(java.util.Collections.emptyList()); + finalInfo.setUploadType(UploadType.CONCATENATED); + UploadInfo createdFinal = storageService.create(finalInfo, null); + + concatenationService.merge(createdFinal); + assertEquals(Long.valueOf(0L), createdFinal.getOffset()); + } }