diff --git a/.github/renovate.json5 b/.github/renovate.json5 index a62871dca4bf..b3aaa9e8cbc2 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -128,6 +128,7 @@ "AutoMapper.Extensions.Microsoft.DependencyInjection", "AWSSDK.SimpleEmail", "AWSSDK.SQS", + "Azure.Storage.Blobs.Batch", "Handlebars.Net", "MailKit", "Microsoft.Azure.NotificationHubs", diff --git a/src/Core/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommand.cs b/src/Core/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommand.cs index f44d464e6cd2..dd2637f4044a 100644 --- a/src/Core/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommand.cs +++ b/src/Core/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommand.cs @@ -7,6 +7,7 @@ using Bit.Core.Exceptions; using Bit.Core.Repositories; using Bit.Core.Services; +using Bit.Core.Tools.Services; using Bit.Core.Vault.Services; using Microsoft.Extensions.Logging; @@ -21,6 +22,7 @@ public class OrganizationDeleteCommand : IOrganizationDeleteCommand private readonly ICipherService _cipherService; private readonly ISubscriberService _subscriberService; private readonly IFeatureService _featureService; + private readonly ISendFileStorageService _sendFileStorageService; private readonly ILogger _logger; public OrganizationDeleteCommand( @@ -31,6 +33,7 @@ public OrganizationDeleteCommand( ICipherService cipherService, ISubscriberService subscriberService, IFeatureService featureService, + ISendFileStorageService sendFileStorageService, ILogger logger) { _applicationCacheService = applicationCacheService; @@ -40,6 +43,7 @@ public OrganizationDeleteCommand( _cipherService = cipherService; _subscriberService = subscriberService; _featureService = featureService; + _sendFileStorageService = sendFileStorageService; _logger = logger; } @@ -70,6 +74,7 @@ public async Task DeleteAsync(Organization organization) } } + await _sendFileStorageService.DeleteFilesForOrganizationAsync(organization.Id); await _cipherService.DeleteAttachmentsForOrganizationAsync(organization.Id); await _organizationRepository.DeleteAsync(organization); await _applicationCacheService.DeleteOrganizationAbilityAsync(organization.Id); diff --git a/src/Core/Core.csproj b/src/Core/Core.csproj index 559ba3386d58..557afce5868b 100644 --- a/src/Core/Core.csproj +++ b/src/Core/Core.csproj @@ -32,6 +32,7 @@ + diff --git a/src/Core/Services/Implementations/UserService.cs b/src/Core/Services/Implementations/UserService.cs index c1a4b078379e..77b260db93da 100644 --- a/src/Core/Services/Implementations/UserService.cs +++ b/src/Core/Services/Implementations/UserService.cs @@ -30,6 +30,7 @@ using Bit.Core.Platform.Push; using Bit.Core.Repositories; using Bit.Core.Settings; +using Bit.Core.Tools.Services; using Bit.Core.Utilities; using Microsoft.AspNetCore.Identity; using Microsoft.Extensions.Caching.Distributed; @@ -69,6 +70,7 @@ public class UserService : UserManager, IUserService private readonly IPricingClient _pricingClient; private readonly IHasPremiumAccessQuery _hasPremiumAccessQuery; private readonly ISubscriberService _subscriberService; + private readonly ISendFileStorageService _sendFileStorageService; public UserService( IUserRepository userRepository, @@ -102,7 +104,8 @@ public UserService( IPolicyRequirementQuery policyRequirementQuery, IPricingClient pricingClient, IHasPremiumAccessQuery hasPremiumAccessQuery, - ISubscriberService subscriberService) + ISubscriberService subscriberService, + ISendFileStorageService sendFileStorageService) : base( store, optionsAccessor, @@ -141,6 +144,7 @@ public UserService( _pricingClient = pricingClient; _hasPremiumAccessQuery = hasPremiumAccessQuery; _subscriberService = subscriberService; + _sendFileStorageService = sendFileStorageService; } public Guid? GetProperUserId(ClaimsPrincipal principal) @@ -237,6 +241,7 @@ public override async Task DeleteAsync(User user) var orgCount = await _organizationUserRepository.GetCountByOrganizationIdAsync(org.Id); if (orgCount <= 1) { + await _sendFileStorageService.DeleteFilesForUserAsync(user.Id); await _organizationRepository.DeleteAsync(org); deletedOrg = true; } @@ -281,6 +286,7 @@ await _subscriberService.CancelSubscription( catch (BillingException) { } } + await _sendFileStorageService.DeleteFilesForUserAsync(user.Id); await _userRepository.DeleteAsync(user); await _pushService.PushLogOutAsync(user.Id); return IdentityResult.Success; diff --git a/src/Core/Tools/Repositories/ISendRepository.cs b/src/Core/Tools/Repositories/ISendRepository.cs index 6de89f03748e..ae334b305d81 100644 --- a/src/Core/Tools/Repositories/ISendRepository.cs +++ b/src/Core/Tools/Repositories/ISendRepository.cs @@ -23,6 +23,18 @@ public interface ISendRepository : IRepository /// Task> GetManyByUserIdAsync(Guid userId); + /// + /// Loads all s owned by an organization. + /// + /// + /// Identifies the organization. + /// + /// + /// A task that completes once the s have been loaded. + /// The task's result contains the loaded s. + /// + Task> GetManyByOrganizationIdAsync(Guid organizationId); + /// /// Loads s scheduled for deletion. /// @@ -35,6 +47,30 @@ public interface ISendRepository : IRepository /// Task> GetManyByDeletionDateAsync(DateTime deletionDateBefore); + /// + /// Loads file-type s created by a user. + /// + /// + /// Identifies the user. + /// + /// + /// A task that completes once the s have been loaded. + /// The task's result contains the loaded file-type s. + /// + Task> GetManyFileSendsByUserIdAsync(Guid userId); + + /// + /// Loads file-type s owned by an organization. + /// + /// + /// Identifies the organization. + /// + /// + /// A task that completes once the s have been loaded. + /// The task's result contains the loaded file-type s. + /// + Task> GetManyFileSendsByOrganizationIdAsync(Guid organizationId); + /// /// Updates encrypted data for sends during a key rotation /// diff --git a/src/Core/Tools/SendFeatures/Commands/NonAnonymousSendCommand.cs b/src/Core/Tools/SendFeatures/Commands/NonAnonymousSendCommand.cs index 21ca1ca3fb00..d9da56ed87bc 100644 --- a/src/Core/Tools/SendFeatures/Commands/NonAnonymousSendCommand.cs +++ b/src/Core/Tools/SendFeatures/Commands/NonAnonymousSendCommand.cs @@ -137,12 +137,22 @@ public async Task UploadFileToExistingSendAsync(Stream stream, Send send) } public async Task DeleteSendAsync(Send send) { - await _sendRepository.DeleteAsync(send); - if (send.Type == Enums.SendType.File) - { - var data = JsonSerializer.Deserialize(send.Data); - await _sendFileStorageService.DeleteFileAsync(send, data.Id); + if (send.Type == Enums.SendType.File && send.Data != null) + { + try + { + var data = send.Data != null ? JsonSerializer.Deserialize(send.Data) : null; + if (data?.Id != null) + { + await _sendFileStorageService.DeleteFileAsync(send, data.Id); + } + } + catch (JsonException ex) + { + _logger.LogWarning(ex, "Failed to deserialize Send {SendId} data; blob may be orphaned.", send.Id); + } } + await _sendRepository.DeleteAsync(send); await _pushNotificationService.PushSyncSendDeleteAsync(send); } diff --git a/src/Core/Tools/SendFeatures/Services/AzureSendFileStorageService.cs b/src/Core/Tools/SendFeatures/Services/AzureSendFileStorageService.cs index 63405b8cdf66..81d1b68bd4d2 100644 --- a/src/Core/Tools/SendFeatures/Services/AzureSendFileStorageService.cs +++ b/src/Core/Tools/SendFeatures/Services/AzureSendFileStorageService.cs @@ -1,21 +1,30 @@ -using Azure.Storage.Blobs; +using System.Text.Json; +using Azure; +using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized; using Azure.Storage.Sas; using Bit.Core.Enums; using Bit.Core.Settings; using Bit.Core.Tools.Entities; +using Bit.Core.Tools.Models.Data; +using Bit.Core.Tools.Repositories; using Microsoft.Extensions.Logging; namespace Bit.Core.Tools.Services; public class AzureSendFileStorageService( GlobalSettings globalSettings, + ISendRepository sendRepository, ILogger logger) : ISendFileStorageService { public const string FilesContainerName = "sendfiles"; private static readonly TimeSpan _downloadLinkLiveTime = TimeSpan.FromMinutes(1); private readonly BlobServiceClient _blobServiceClient = new(globalSettings.Send.ConnectionString); + private readonly ISendRepository _sendRepository = sendRepository; + private readonly ILogger _logger = logger; + /* * When this file was made nullable, multiple instances of ! were introduced asserting that * _sendFilesContainerClient abd the blobClient it is used to construct are not null. @@ -46,10 +55,7 @@ public async Task UploadNewFileAsync(Stream stream, Send send, string fileId) metadata.Add("organizationId", send.OrganizationId.Value.ToString()); } - var headers = new BlobHttpHeaders - { - ContentDisposition = $"attachment; filename=\"{fileId}\"" - }; + var headers = new BlobHttpHeaders { ContentDisposition = $"attachment; filename=\"{fileId}\"" }; await blobClient.UploadAsync(stream, new BlobUploadOptions { Metadata = metadata, HttpHeaders = headers }); } @@ -66,11 +72,15 @@ public async Task DeleteBlobAsync(string blobName) public async Task DeleteFilesForOrganizationAsync(Guid organizationId) { await InitAsync(); + var sends = await _sendRepository.GetManyFileSendsByOrganizationIdAsync(organizationId); + await DeleteBlobsForSendsAsync(sends); } public async Task DeleteFilesForUserAsync(Guid userId) { await InitAsync(); + var sends = await _sendRepository.GetManyFileSendsByUserIdAsync(userId); + await DeleteBlobsForSendsAsync(sends); } public async Task GetSendFileDownloadUrlAsync(Send send, string fileId) @@ -85,7 +95,8 @@ public async Task GetSendFileUploadUrlAsync(Send send, string fileId) { await InitAsync(); var blobClient = _sendFilesContainerClient!.GetBlobClient(BlobName(send, fileId)); - var sasUri = blobClient.GenerateSasUri(BlobSasPermissions.Create | BlobSasPermissions.Write, DateTime.UtcNow.Add(_downloadLinkLiveTime)); + var sasUri = blobClient.GenerateSasUri(BlobSasPermissions.Create | BlobSasPermissions.Write, + DateTime.UtcNow.Add(_downloadLinkLiveTime)); return sasUri.ToString(); } @@ -108,12 +119,10 @@ public async Task GetSendFileUploadUrlAsync(Send send, string fileId) { metadata["organizationId"] = send.OrganizationId.Value.ToString(); } + await blobClient.SetMetadataAsync(metadata); - var headers = new BlobHttpHeaders - { - ContentDisposition = $"attachment; filename=\"{fileId}\"" - }; + var headers = new BlobHttpHeaders { ContentDisposition = $"attachment; filename=\"{fileId}\"" }; await blobClient.SetHttpHeadersAsync(headers); var length = blobProperties.Value.ContentLength; @@ -128,6 +137,58 @@ public async Task GetSendFileUploadUrlAsync(Send send, string fileId) } } + private async Task DeleteBlobsForSendsAsync(ICollection fileSends) + { + var blobUris = new List(); + + foreach (var send in fileSends) + { + try + { + var data = send.Data != null + ? JsonSerializer.Deserialize(send.Data) + : null; + if (data?.Id != null) + { + var blobClient = _sendFilesContainerClient!.GetBlobClient(BlobName(send, data.Id)); + blobUris.Add(blobClient.Uri); + } + } + catch (JsonException ex) + { + _logger.LogWarning(Constants.BypassFiltersEventId, ex, + "Failed to deserialize Send {SendId} data; blob may be orphaned.", send.Id); + } + } + + if (blobUris.Count == 0) + { + return; + } + + var blobBatchClient = _blobServiceClient.GetBlobBatchClient(); + + foreach (var batch in blobUris.Chunk(256)) + { + try + { + await blobBatchClient.DeleteBlobsAsync(batch); + } + catch (AggregateException ex) + { + _logger.LogError(Constants.BypassFiltersEventId, ex, + "One or more blob deletions failed in a batch of {Count} blobs. The following URIs may be orphaned: {BlobUris}", + batch.Length, string.Join(", ", batch)); + } + catch (RequestFailedException ex) + { + _logger.LogError(Constants.BypassFiltersEventId, ex, + "Batch blob deletion request failed for {Count} blobs. The following URIs may be orphaned: {BlobUris}", + batch.Length, string.Join(", ", batch)); + } + } + } + private async Task InitAsync() { if (_sendFilesContainerClient == null) diff --git a/src/Core/Tools/SendFeatures/Services/LocalSendStorageService.cs b/src/Core/Tools/SendFeatures/Services/LocalSendStorageService.cs index abe22a28e284..3a1aaf7478d5 100644 --- a/src/Core/Tools/SendFeatures/Services/LocalSendStorageService.cs +++ b/src/Core/Tools/SendFeatures/Services/LocalSendStorageService.cs @@ -1,14 +1,22 @@ -using Bit.Core.Enums; +using System.Text.Json; +using Bit.Core.Enums; using Bit.Core.Settings; using Bit.Core.Tools.Entities; +using Bit.Core.Tools.Models.Data; +using Bit.Core.Tools.Repositories; +using Microsoft.Extensions.Logging; namespace Bit.Core.Tools.Services; public class LocalSendStorageService( - GlobalSettings globalSettings) : ISendFileStorageService + GlobalSettings globalSettings, + ISendRepository sendRepository, + ILogger logger) : ISendFileStorageService { private readonly string _baseDirPath = globalSettings.Send.BaseDirectory; private readonly string _baseSendUrl = globalSettings.Send.BaseUrl; + private readonly ISendRepository _sendRepository = sendRepository; + private readonly ILogger _logger = logger; private string RelativeFilePath(Send send, string fileID) => $"{send.Id}/{fileID}"; private string FilePath(Send send, string fileID) => $"{_baseDirPath}/{RelativeFilePath(send, fileID)}"; public FileUploadType FileUploadType => FileUploadType.Direct; @@ -40,11 +48,15 @@ public async Task DeleteFileAsync(Send send, string fileId) public async Task DeleteFilesForOrganizationAsync(Guid organizationId) { await InitAsync(); + var sends = await _sendRepository.GetManyFileSendsByOrganizationIdAsync(organizationId); + await DeleteFilesForSendsAsync(sends); } public async Task DeleteFilesForUserAsync(Guid userId) { await InitAsync(); + var sends = await _sendRepository.GetManyFileSendsByUserIdAsync(userId); + await DeleteFilesForSendsAsync(sends); } public async Task GetSendFileDownloadUrlAsync(Send send, string fileId) @@ -53,6 +65,27 @@ public async Task GetSendFileDownloadUrlAsync(Send send, string fileId) return $"{_baseSendUrl}/{RelativeFilePath(send, fileId)}"; } + private async Task DeleteFilesForSendsAsync(ICollection sends) + { + foreach (var send in sends) + { + try + { + var data = send.Data != null + ? JsonSerializer.Deserialize(send.Data) + : null; + if (data?.Id != null) + { + await DeleteFileAsync(send, data.Id); + } + } + catch (JsonException ex) + { + _logger.LogWarning(ex, "Failed to deserialize Send {SendId} data; blob may be orphaned.", send.Id); + } + } + } + private void DeleteFileIfExists(string path) { if (File.Exists(path)) diff --git a/src/Infrastructure.Dapper/Tools/Repositories/SendRepository.cs b/src/Infrastructure.Dapper/Tools/Repositories/SendRepository.cs index 144e08021d71..a8c70810affe 100644 --- a/src/Infrastructure.Dapper/Tools/Repositories/SendRepository.cs +++ b/src/Infrastructure.Dapper/Tools/Repositories/SendRepository.cs @@ -52,6 +52,54 @@ public async Task> GetManyByUserIdAsync(Guid userId) } } + /// + public async Task> GetManyByOrganizationIdAsync(Guid organizationId) + { + using (var connection = new SqlConnection(ConnectionString)) + { + var results = await connection.QueryAsync( + $"[{Schema}].[Send_ReadByOrganizationId]", + new { OrganizationId = organizationId }, + commandType: CommandType.StoredProcedure); + + var sends = results.ToList(); + UnprotectData(sends); + return sends; + } + } + + /// + public async Task> GetManyFileSendsByUserIdAsync(Guid userId) + { + using (var connection = new SqlConnection(ConnectionString)) + { + var results = await connection.QueryAsync( + $"[{Schema}].[Send_ReadFilesByUserId]", + new { UserId = userId }, + commandType: CommandType.StoredProcedure); + + var sends = results.ToList(); + UnprotectData(sends); + return sends; + } + } + + /// + public async Task> GetManyFileSendsByOrganizationIdAsync(Guid organizationId) + { + using (var connection = new SqlConnection(ConnectionString)) + { + var results = await connection.QueryAsync( + $"[{Schema}].[Send_ReadFilesByOrganizationId]", + new { OrganizationId = organizationId }, + commandType: CommandType.StoredProcedure); + + var sends = results.ToList(); + UnprotectData(sends); + return sends; + } + } + /// public async Task> GetManyByDeletionDateAsync(DateTime deletionDateBefore) { diff --git a/src/Infrastructure.EntityFramework/AdminConsole/Repositories/OrganizationRepository.cs b/src/Infrastructure.EntityFramework/AdminConsole/Repositories/OrganizationRepository.cs index 48dd04f62af4..8e72fe78abaf 100644 --- a/src/Infrastructure.EntityFramework/AdminConsole/Repositories/OrganizationRepository.cs +++ b/src/Infrastructure.EntityFramework/AdminConsole/Repositories/OrganizationRepository.cs @@ -267,6 +267,9 @@ await dbContext.NotificationStatuses.Where(ns => ns.Notification.OrganizationId await dbContext.Notifications.Where(n => n.OrganizationId == organization.Id) .ExecuteDeleteAsync(); + await dbContext.Sends.Where(s => s.OrganizationId == organization.Id) + .ExecuteDeleteAsync(); + // The below section are 3 SPROCS in SQL Server but are only called by here await dbContext.OrganizationApiKeys.Where(oa => oa.OrganizationId == organization.Id) .ExecuteDeleteAsync(); diff --git a/src/Infrastructure.EntityFramework/Tools/Repositories/SendRepository.cs b/src/Infrastructure.EntityFramework/Tools/Repositories/SendRepository.cs index adf3fcc1f1cd..9a22e48609c2 100644 --- a/src/Infrastructure.EntityFramework/Tools/Repositories/SendRepository.cs +++ b/src/Infrastructure.EntityFramework/Tools/Repositories/SendRepository.cs @@ -2,6 +2,7 @@ using AutoMapper; using Bit.Core.KeyManagement.UserKey; +using Bit.Core.Tools.Enums; using Bit.Core.Tools.Repositories; using Bit.Infrastructure.EntityFramework.Models; using Bit.Infrastructure.EntityFramework.Repositories; @@ -71,6 +72,43 @@ public SendRepository(IServiceScopeFactory serviceScopeFactory, IMapper mapper) } } + /// + public async Task> GetManyByOrganizationIdAsync(Guid organizationId) + { + using (var scope = ServiceScopeFactory.CreateScope()) + { + var dbContext = GetDatabaseContext(scope); + var results = await dbContext.Sends.Where(s => s.OrganizationId == organizationId).ToListAsync(); + return Mapper.Map>(results); + } + } + + /// + public async Task> GetManyFileSendsByUserIdAsync(Guid userId) + { + using (var scope = ServiceScopeFactory.CreateScope()) + { + var dbContext = GetDatabaseContext(scope); + var results = await dbContext.Sends + .Where(s => s.UserId == userId && s.Type == SendType.File) + .ToListAsync(); + return Mapper.Map>(results); + } + } + + /// + public async Task> GetManyFileSendsByOrganizationIdAsync(Guid organizationId) + { + using (var scope = ServiceScopeFactory.CreateScope()) + { + var dbContext = GetDatabaseContext(scope); + var results = await dbContext.Sends + .Where(s => s.OrganizationId == organizationId && s.Type == SendType.File) + .ToListAsync(); + return Mapper.Map>(results); + } + } + /// public UpdateEncryptedDataForKeyRotation UpdateForKeyRotation(Guid userId, IEnumerable sends) diff --git a/src/Sql/dbo/Stored Procedures/Organization_DeleteById.sql b/src/Sql/dbo/Stored Procedures/Organization_DeleteById.sql index 50ad247c1afb..87d14a1013f6 100644 --- a/src/Sql/dbo/Stored Procedures/Organization_DeleteById.sql +++ b/src/Sql/dbo/Stored Procedures/Organization_DeleteById.sql @@ -151,6 +151,13 @@ BEGIN WHERE [OrganizationId] = @Id + -- Delete Organization Owned Sends + DELETE + FROM + [dbo].[Send] + WHERE + [OrganizationId] = @Id + DELETE FROM [dbo].[Organization] diff --git a/src/Sql/dbo/Tools/Stored Procedures/Send_ReadByOrganizationId.sql b/src/Sql/dbo/Tools/Stored Procedures/Send_ReadByOrganizationId.sql new file mode 100644 index 000000000000..81b1647ebefe --- /dev/null +++ b/src/Sql/dbo/Tools/Stored Procedures/Send_ReadByOrganizationId.sql @@ -0,0 +1,13 @@ +CREATE PROCEDURE [dbo].[Send_ReadByOrganizationId] + @OrganizationId UNIQUEIDENTIFIER +AS +BEGIN + SET NOCOUNT ON + + SELECT + * + FROM + [dbo].[SendView] + WHERE + [OrganizationId] = @OrganizationId +END diff --git a/src/Sql/dbo/Tools/Stored Procedures/Send_ReadFilesByOrganizationId.sql b/src/Sql/dbo/Tools/Stored Procedures/Send_ReadFilesByOrganizationId.sql new file mode 100644 index 000000000000..9b0a812e3409 --- /dev/null +++ b/src/Sql/dbo/Tools/Stored Procedures/Send_ReadFilesByOrganizationId.sql @@ -0,0 +1,14 @@ +CREATE PROCEDURE [dbo].[Send_ReadFilesByOrganizationId] + @OrganizationId UNIQUEIDENTIFIER +AS +BEGIN + SET NOCOUNT ON + + SELECT + * + FROM + [dbo].[SendView] + WHERE + [OrganizationId] = @OrganizationId + AND [Type] = 1 +END diff --git a/src/Sql/dbo/Tools/Stored Procedures/Send_ReadFilesByUserId.sql b/src/Sql/dbo/Tools/Stored Procedures/Send_ReadFilesByUserId.sql new file mode 100644 index 000000000000..bfb993effd79 --- /dev/null +++ b/src/Sql/dbo/Tools/Stored Procedures/Send_ReadFilesByUserId.sql @@ -0,0 +1,15 @@ +CREATE PROCEDURE [dbo].[Send_ReadFilesByUserId] + @UserId UNIQUEIDENTIFIER +AS +BEGIN + SET NOCOUNT ON + + SELECT + * + FROM + [dbo].[SendView] + WHERE + [OrganizationId] IS NULL + AND [UserId] = @UserId + AND [Type] = 1 +END diff --git a/test/Core.Test/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommandTests.cs b/test/Core.Test/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommandTests.cs index e53151ce6cc7..9eed12224f17 100644 --- a/test/Core.Test/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommandTests.cs +++ b/test/Core.Test/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommandTests.cs @@ -10,6 +10,7 @@ using Bit.Core.Repositories; using Bit.Core.Services; using Bit.Core.Test.AutoFixture.OrganizationFixtures; +using Bit.Core.Tools.Services; using Bit.Core.Vault.Services; using Bit.Test.Common.AutoFixture; using Bit.Test.Common.AutoFixture.Attributes; @@ -148,4 +149,32 @@ public async Task Delete_WhenFlagDisabled_HandlesBillingException( await sutProvider.GetDependency().Received(1).DeleteAsync(organization); } + + [Theory, PaidOrganizationCustomize, BitAutoData] + public async Task Delete_WithFileSends_DeletesFilesBeforeDbRecords( + Organization organization, + SutProvider sutProvider) + { + // Ensuring that the file is deleted first avoids the following situation: + // 1. DB row is deleted successfully + // 2. File blob fails to delete + // 3. File blob still exists but with no parent Send + var callOrder = new List(); + sutProvider.GetDependency() + .DeleteFilesForOrganizationAsync(organization.Id) + .Returns(Task.CompletedTask) + .AndDoes(_ => callOrder.Add("file")); + sutProvider.GetDependency() + .DeleteAsync(organization) + .Returns(Task.CompletedTask) + .AndDoes(_ => callOrder.Add("db")); + + await sutProvider.Sut.DeleteAsync(organization); + + await sutProvider.GetDependency() + .Received(1).DeleteFilesForOrganizationAsync(organization.Id); + await sutProvider.GetDependency() + .Received(1).DeleteAsync(organization); + Assert.Equal(new[] { "file", "db" }, callOrder); + } } diff --git a/test/Core.Test/Services/UserServiceTests.cs b/test/Core.Test/Services/UserServiceTests.cs index 1a1425d3dc1f..1dc59ba38ca4 100644 --- a/test/Core.Test/Services/UserServiceTests.cs +++ b/test/Core.Test/Services/UserServiceTests.cs @@ -25,6 +25,7 @@ using Bit.Core.Services; using Bit.Core.Settings; using Bit.Core.Test.AdminConsole.AutoFixture; +using Bit.Core.Tools.Services; using Bit.Core.Utilities; using Bit.Test.Common.AutoFixture; using Bit.Test.Common.AutoFixture.Attributes; @@ -699,6 +700,43 @@ await sutProvider.GetDependency() .DidNotReceiveWithAnyArgs() .CancelSubscription(default, default, default); } + + [Theory, BitAutoData] + public async Task DeleteAsync_WithFileSends_DeletesFilesBeforeDbRecords( + User user, + SutProvider sutProvider) + { + // Ensuring that the file is deleted first avoids the following situation: + // 1. DB row is deleted successfully + // 2. File blob fails to delete + // 3. File blob still exists but with no parent Send + user.GatewaySubscriptionId = null; + + sutProvider.GetDependency() + .GetCountByOnlyOwnerAsync(user.Id) + .Returns(0); + + sutProvider.GetDependency() + .GetCountByOnlyOwnerAsync(user.Id) + .Returns(0); + + var callOrder = new List(); + sutProvider.GetDependency() + .DeleteFilesForUserAsync(user.Id) + .Returns(Task.CompletedTask) + .AndDoes(_ => callOrder.Add("file")); + sutProvider.GetDependency() + .DeleteAsync(user) + .Returns(Task.CompletedTask) + .AndDoes(_ => callOrder.Add("db")); + + var result = await sutProvider.Sut.DeleteAsync(user); + + Assert.True(result.Succeeded); + await sutProvider.GetDependency() + .Received(1).DeleteFilesForUserAsync(user.Id); + Assert.Equal(new[] { "file", "db" }, callOrder); + } } public static class UserServiceSutProviderExtensions diff --git a/test/Core.Test/Tools/Services/LocalSendStorageServiceTests.cs b/test/Core.Test/Tools/Services/LocalSendStorageServiceTests.cs new file mode 100644 index 000000000000..9a490737f8e9 --- /dev/null +++ b/test/Core.Test/Tools/Services/LocalSendStorageServiceTests.cs @@ -0,0 +1,194 @@ +using System.Text.Json; +using Bit.Core.Tools.Entities; +using Bit.Core.Tools.Enums; +using Bit.Core.Tools.Models.Data; +using Bit.Core.Tools.Repositories; +using Bit.Core.Tools.Services; +using Microsoft.Extensions.Logging; +using NSubstitute; +using Xunit; + +namespace Bit.Core.Test.Tools.Services; + +public class LocalSendStorageServiceTests +{ + [Fact] + public async Task DeleteFileAsync_FileExists_DeletesFileAndEmptyDirectory() + { + using var tempDirectory = new TempDirectory(); + var sut = CreateSut(tempDirectory); + + var send = new Send { Id = Guid.NewGuid(), Type = SendType.File }; + var fileId = "testfile"; + + // Create the file on disk + var dirPath = Path.Combine(tempDirectory.Directory, send.Id.ToString()); + Directory.CreateDirectory(dirPath); + var filePath = Path.Combine(dirPath, fileId); + await File.WriteAllTextAsync(filePath, "file contents"); + + await sut.DeleteFileAsync(send, fileId); + + Assert.False(File.Exists(filePath)); + Assert.False(Directory.Exists(dirPath)); + } + + [Fact] + public async Task DeleteFileAsync_FileDoesNotExist_DoesNotThrow() + { + using var tempDirectory = new TempDirectory(); + var sut = CreateSut(tempDirectory); + + var send = new Send { Id = Guid.NewGuid(), Type = SendType.File }; + + await sut.DeleteFileAsync(send, "nonexistent"); + } + + [Fact] + public async Task DeleteFileAsync_DirectoryHasOtherFiles_KeepsDirectory() + { + using var tempDirectory = new TempDirectory(); + var sut = CreateSut(tempDirectory); + + var send = new Send { Id = Guid.NewGuid(), Type = SendType.File }; + var fileId = "testfile"; + + var dirPath = Path.Combine(tempDirectory.Directory, send.Id.ToString()); + Directory.CreateDirectory(dirPath); + await File.WriteAllTextAsync(Path.Combine(dirPath, fileId), "delete me"); + await File.WriteAllTextAsync(Path.Combine(dirPath, "otherfile"), "keep me"); + + await sut.DeleteFileAsync(send, fileId); + + Assert.False(File.Exists(Path.Combine(dirPath, fileId))); + Assert.True(Directory.Exists(dirPath)); + Assert.True(File.Exists(Path.Combine(dirPath, "otherfile"))); + } + + [Fact] + public async Task DeleteFilesForUserAsync_WithFileSends_DeletesFiles() + { + using var tempDirectory = new TempDirectory(); + var sendRepository = Substitute.For(); + var sut = CreateSut(tempDirectory, sendRepository); + + var userId = Guid.NewGuid(); + var send1 = new Send { Id = Guid.NewGuid(), UserId = userId, Type = SendType.File }; + var send2 = new Send { Id = Guid.NewGuid(), UserId = userId, Type = SendType.File }; + var fileId1 = "file1"; + var fileId2 = "file2"; + send1.Data = JsonSerializer.Serialize(new SendFileData { Id = fileId1, FileName = "a.txt" }); + send2.Data = JsonSerializer.Serialize(new SendFileData { Id = fileId2, FileName = "b.txt" }); + + // Create files on disk + var dir1 = Path.Combine(tempDirectory.Directory, send1.Id.ToString()); + var dir2 = Path.Combine(tempDirectory.Directory, send2.Id.ToString()); + Directory.CreateDirectory(dir1); + Directory.CreateDirectory(dir2); + await File.WriteAllTextAsync(Path.Combine(dir1, fileId1), "contents1"); + await File.WriteAllTextAsync(Path.Combine(dir2, fileId2), "contents2"); + + sendRepository.GetManyFileSendsByUserIdAsync(userId).Returns(new List { send1, send2 }); + + await sut.DeleteFilesForUserAsync(userId); + + Assert.False(File.Exists(Path.Combine(dir1, fileId1))); + Assert.False(Directory.Exists(dir1)); + Assert.False(File.Exists(Path.Combine(dir2, fileId2))); + Assert.False(Directory.Exists(dir2)); + } + + [Fact] + public async Task DeleteFilesForUserAsync_NoFileSends_DoesNothing() + { + using var tempDirectory = new TempDirectory(); + var sendRepository = Substitute.For(); + var sut = CreateSut(tempDirectory, sendRepository); + + var userId = Guid.NewGuid(); + var textSend = new Send { Id = Guid.NewGuid(), UserId = userId, Type = SendType.Text }; + + sendRepository.GetManyFileSendsByUserIdAsync(userId).Returns(new List { textSend }); + + await sut.DeleteFilesForUserAsync(userId); + + // No files should have been touched — base directory should still be empty + Assert.Empty(Directory.GetDirectories(tempDirectory.Directory)); + } + + [Fact] + public async Task DeleteFilesForUserAsync_NoSends_DoesNotThrow() + { + using var tempDirectory = new TempDirectory(); + var sendRepository = Substitute.For(); + var sut = CreateSut(tempDirectory, sendRepository); + + var userId = Guid.NewGuid(); + sendRepository.GetManyFileSendsByUserIdAsync(userId).Returns(new List()); + + await sut.DeleteFilesForUserAsync(userId); + } + + [Fact] + public async Task DeleteFilesForUserAsync_MalformedData_LogsWarningAndContinues() + { + using var tempDirectory = new TempDirectory(); + var sendRepository = Substitute.For(); + var sut = CreateSut(tempDirectory, sendRepository); + + var userId = Guid.NewGuid(); + var badSend = new Send { Id = Guid.NewGuid(), UserId = userId, Type = SendType.File, Data = "not valid json{{{" }; + var goodSend = new Send { Id = Guid.NewGuid(), UserId = userId, Type = SendType.File }; + var fileId = "goodfile"; + goodSend.Data = JsonSerializer.Serialize(new SendFileData { Id = fileId, FileName = "c.txt" }); + + // Create file for the good send + var dir = Path.Combine(tempDirectory.Directory, goodSend.Id.ToString()); + Directory.CreateDirectory(dir); + await File.WriteAllTextAsync(Path.Combine(dir, fileId), "contents"); + + sendRepository.GetManyFileSendsByUserIdAsync(userId).Returns(new List { badSend, goodSend }); + + await sut.DeleteFilesForUserAsync(userId); + + // Good send's file should still be deleted despite the bad send + Assert.False(File.Exists(Path.Combine(dir, fileId))); + Assert.False(Directory.Exists(dir)); + } + + [Fact] + public async Task DeleteFilesForOrganizationAsync_WithFileSends_DeletesFiles() + { + using var tempDirectory = new TempDirectory(); + var sendRepository = Substitute.For(); + var sut = CreateSut(tempDirectory, sendRepository); + + var orgId = Guid.NewGuid(); + var send = new Send { Id = Guid.NewGuid(), OrganizationId = orgId, Type = SendType.File }; + var fileId = "orgfile"; + send.Data = JsonSerializer.Serialize(new SendFileData { Id = fileId, FileName = "d.txt" }); + + var dir = Path.Combine(tempDirectory.Directory, send.Id.ToString()); + Directory.CreateDirectory(dir); + await File.WriteAllTextAsync(Path.Combine(dir, fileId), "org contents"); + + sendRepository.GetManyFileSendsByOrganizationIdAsync(orgId).Returns(new List { send }); + + await sut.DeleteFilesForOrganizationAsync(orgId); + + Assert.False(File.Exists(Path.Combine(dir, fileId))); + Assert.False(Directory.Exists(dir)); + } + + private static LocalSendStorageService CreateSut(TempDirectory tempDirectory, + ISendRepository? sendRepository = null) + { + var globalSettings = new Bit.Core.Settings.GlobalSettings(); + globalSettings.Send.BaseDirectory = tempDirectory.Directory; + globalSettings.Send.BaseUrl = "https://localhost/sends"; + return new LocalSendStorageService( + globalSettings, + sendRepository ?? Substitute.For(), + Substitute.For>()); + } +} diff --git a/test/Core.Test/Tools/Services/NonAnonymousSendCommandTests.cs b/test/Core.Test/Tools/Services/NonAnonymousSendCommandTests.cs index ecb9cdba2ced..a81cd9d8197b 100644 --- a/test/Core.Test/Tools/Services/NonAnonymousSendCommandTests.cs +++ b/test/Core.Test/Tools/Services/NonAnonymousSendCommandTests.cs @@ -1404,4 +1404,36 @@ public void SendCanBeAccessed_WithNullExpirationDate_ReturnsTrue() // Assert Assert.True(result); } + + [Fact] + public async Task DeleteSendAsync_FileSend_DeletesFileBeforeDbRecord() + { + // Ensuring that the file is deleted first avoids the following situation: + // 1. DB row is deleted successfully + // 2. File blob fails to delete + // 3. File blob still exists but with no parent Send + var fileData = new SendFileData { Id = "file123", FileName = "test.txt", Size = 100 }; + var send = new Send + { + Id = Guid.NewGuid(), + Type = SendType.File, + Data = JsonSerializer.Serialize(fileData), + UserId = Guid.NewGuid() + }; + + var callOrder = new List(); + _sendFileStorageService.DeleteFileAsync(send, fileData.Id) + .Returns(Task.CompletedTask) + .AndDoes(_ => callOrder.Add("file")); + _sendRepository.DeleteAsync(send) + .Returns(Task.CompletedTask) + .AndDoes(_ => callOrder.Add("db")); + + await _nonAnonymousSendCommand.DeleteSendAsync(send); + + await _sendFileStorageService.Received(1).DeleteFileAsync(send, fileData.Id); + await _sendRepository.Received(1).DeleteAsync(send); + await _pushNotificationService.Received(1).PushSyncSendDeleteAsync(send); + Assert.Equal(new[] { "file", "db" }, callOrder); + } } diff --git a/util/Migrator/DbScripts/2026-04-13_00_OrganizationDeleteById_AddSendDeletion.sql b/util/Migrator/DbScripts/2026-04-13_00_OrganizationDeleteById_AddSendDeletion.sql new file mode 100644 index 000000000000..7d898d8c95a2 --- /dev/null +++ b/util/Migrator/DbScripts/2026-04-13_00_OrganizationDeleteById_AddSendDeletion.sql @@ -0,0 +1,169 @@ +CREATE OR ALTER PROCEDURE [dbo].[Organization_DeleteById] + @Id UNIQUEIDENTIFIER +WITH RECOMPILE +AS +BEGIN + SET NOCOUNT ON + + EXEC [dbo].[User_BumpAccountRevisionDateByOrganizationId] @Id + + DECLARE @BatchSize INT = 100 + WHILE @BatchSize > 0 + BEGIN + BEGIN TRANSACTION Organization_DeleteById_Ciphers + + DELETE TOP(@BatchSize) + FROM + [dbo].[Cipher] + WHERE + [UserId] IS NULL + AND [OrganizationId] = @Id + + SET @BatchSize = @@ROWCOUNT + + COMMIT TRANSACTION Organization_DeleteById_Ciphers + END + + BEGIN TRANSACTION Organization_DeleteById + + DELETE + FROM + [dbo].[AuthRequest] + WHERE + [OrganizationId] = @Id + + DELETE + FROM + [dbo].[SsoUser] + WHERE + [OrganizationId] = @Id + + DELETE + FROM + [dbo].[SsoConfig] + WHERE + [OrganizationId] = @Id + + DELETE CU + FROM + [dbo].[CollectionUser] CU + INNER JOIN + [dbo].[OrganizationUser] OU ON [CU].[OrganizationUserId] = [OU].[Id] + WHERE + [OU].[OrganizationId] = @Id + + DELETE AP + FROM + [dbo].[AccessPolicy] AP + INNER JOIN + [dbo].[OrganizationUser] OU ON [AP].[OrganizationUserId] = [OU].[Id] + WHERE + [OU].[OrganizationId] = @Id + + DELETE GU + FROM + [dbo].[GroupUser] GU + INNER JOIN + [dbo].[OrganizationUser] OU ON [GU].[OrganizationUserId] = [OU].[Id] + WHERE + [OU].[OrganizationId] = @Id + + DELETE + FROM + [dbo].[OrganizationUser] + WHERE + [OrganizationId] = @Id + + DELETE + FROM + [dbo].[ProviderOrganization] + WHERE + [OrganizationId] = @Id + + EXEC [dbo].[OrganizationApiKey_OrganizationDeleted] @Id + EXEC [dbo].[OrganizationConnection_OrganizationDeleted] @Id + EXEC [dbo].[OrganizationSponsorship_OrganizationDeleted] @Id + EXEC [dbo].[OrganizationDomain_OrganizationDeleted] @Id + EXEC [dbo].[OrganizationIntegration_OrganizationDeleted] @Id + + DELETE + FROM + [dbo].[Project] + WHERE + [OrganizationId] = @Id + + DELETE + FROM + [dbo].[Secret] + WHERE + [OrganizationId] = @Id + + DELETE AK + FROM + [dbo].[ApiKey] AK + INNER JOIN + [dbo].[ServiceAccount] SA ON [AK].[ServiceAccountId] = [SA].[Id] + WHERE + [SA].[OrganizationId] = @Id + + DELETE AP + FROM + [dbo].[AccessPolicy] AP + INNER JOIN + [dbo].[ServiceAccount] SA ON [AP].[GrantedServiceAccountId] = [SA].[Id] + WHERE + [SA].[OrganizationId] = @Id + + DELETE + FROM + [dbo].[ServiceAccount] + WHERE + [OrganizationId] = @Id + + -- Delete Notification Status + DELETE + NS + FROM + [dbo].[NotificationStatus] NS + INNER JOIN + [dbo].[Notification] N ON N.[Id] = NS.[NotificationId] + WHERE + N.[OrganizationId] = @Id + + -- Delete Notification + DELETE + FROM + [dbo].[Notification] + WHERE + [OrganizationId] = @Id + + -- Delete Organization Application + DELETE + FROM + [dbo].[OrganizationApplication] + WHERE + [OrganizationId] = @Id + + -- Delete Organization Report + DELETE + FROM + [dbo].[OrganizationReport] + WHERE + [OrganizationId] = @Id + + -- Delete Organization Owned Sends + DELETE + FROM + [dbo].[Send] + WHERE + [OrganizationId] = @Id + + DELETE + FROM + [dbo].[Organization] + WHERE + [Id] = @Id + + COMMIT TRANSACTION Organization_DeleteById +END +GO diff --git a/util/Migrator/DbScripts/2026-04-13_01_AddSendReadByOrganizationId.sql b/util/Migrator/DbScripts/2026-04-13_01_AddSendReadByOrganizationId.sql new file mode 100644 index 000000000000..b1aa1bcdfbda --- /dev/null +++ b/util/Migrator/DbScripts/2026-04-13_01_AddSendReadByOrganizationId.sql @@ -0,0 +1,14 @@ +CREATE OR ALTER PROCEDURE [dbo].[Send_ReadByOrganizationId] + @OrganizationId UNIQUEIDENTIFIER +AS +BEGIN + SET NOCOUNT ON + + SELECT + * + FROM + [dbo].[SendView] + WHERE + [OrganizationId] = @OrganizationId +END +GO diff --git a/util/Migrator/DbScripts/2026-04-13_02_SendReadFilesByUserIdAndOrganizationId.sql b/util/Migrator/DbScripts/2026-04-13_02_SendReadFilesByUserIdAndOrganizationId.sql new file mode 100644 index 000000000000..21aea17b70a9 --- /dev/null +++ b/util/Migrator/DbScripts/2026-04-13_02_SendReadFilesByUserIdAndOrganizationId.sql @@ -0,0 +1,32 @@ +CREATE OR ALTER PROCEDURE [dbo].[Send_ReadFilesByUserId] + @UserId UNIQUEIDENTIFIER +AS +BEGIN + SET NOCOUNT ON + + SELECT + * + FROM + [dbo].[SendView] + WHERE + [OrganizationId] IS NULL + AND [UserId] = @UserId + AND [Type] = 1 +END +GO + +CREATE OR ALTER PROCEDURE [dbo].[Send_ReadFilesByOrganizationId] + @OrganizationId UNIQUEIDENTIFIER +AS +BEGIN + SET NOCOUNT ON + + SELECT + * + FROM + [dbo].[SendView] + WHERE + [OrganizationId] = @OrganizationId + AND [Type] = 1 +END +GO