Browse Source

Merge upstream 0.19

master
Pasture Pastureson 3 weeks ago
parent
commit
f6acf7f277
43 changed files with 1173 additions and 103 deletions
  1. +4
    -0
      VARIABLES.md
  2. +10
    -2
      src/BSLManager/App.cs
  3. +1
    -1
      src/BSLManager/Domain/FollowersListState.cs
  4. +4
    -1
      src/BirdsiteLive.Common/Settings/InstanceSettings.cs
  5. +11
    -1
      src/BirdsiteLive.Domain/Repository/PublicationRepository.cs
  6. +8
    -2
      src/BirdsiteLive.Domain/StatusService.cs
  7. +1
    -0
      src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj
  8. +12
    -0
      src/BirdsiteLive.Pipeline/Contracts/IRefreshTwitterUserStatusProcessor.cs
  9. +1
    -1
      src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs
  10. +1
    -1
      src/BirdsiteLive.Pipeline/Contracts/IRetrieveTweetsProcessor.cs
  11. +1
    -1
      src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs
  12. +1
    -1
      src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs
  13. +1
    -1
      src/BirdsiteLive.Pipeline/Models/UserWithDataToSync.cs
  14. +74
    -0
      src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs
  15. +1
    -1
      src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs
  16. +9
    -16
      src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs
  17. +6
    -1
      src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
  18. +2
    -2
      src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs
  19. +31
    -6
      src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
  20. +18
    -10
      src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs
  21. +1
    -1
      src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs
  22. +3
    -0
      src/BirdsiteLive.Twitter/TwitterUserService.cs
  23. +1
    -1
      src/BirdsiteLive/BirdsiteLive.csproj
  24. +3
    -1
      src/BirdsiteLive/Controllers/StatisticsController.cs
  25. +1
    -0
      src/BirdsiteLive/Controllers/UsersController.cs
  26. +2
    -0
      src/BirdsiteLive/Models/StatisticsModels/Statistics.cs
  27. +2
    -0
      src/BirdsiteLive/Views/Statistics/Index.cshtml
  28. +3
    -1
      src/BirdsiteLive/appsettings.json
  29. +14
    -2
      src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs
  30. +18
    -3
      src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs
  31. +21
    -3
      src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
  32. +1
    -0
      src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs
  33. +3
    -1
      src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
  34. +2
    -0
      src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs
  35. +2
    -0
      src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs
  36. +91
    -3
      src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs
  37. +64
    -6
      src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs
  38. +333
    -0
      src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs
  39. +3
    -3
      src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs
  40. +22
    -14
      src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs
  41. +6
    -3
      src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs
  42. +378
    -12
      src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs
  43. +2
    -1
      src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs

+ 4
- 0
VARIABLES.md View File

@ -53,6 +53,9 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act
* `Instance:ShowAboutInstanceOnProfiles` (default: true) show "About [instance name]" on profiles with a link to /About
* `Instance:MaxFollowsPerUser` (default: 0 - no limit) limit the number of follows per user - any follow count above this number will be Rejected
* `Instance:DiscloseInstanceRestrictions` (default: false) disclose your instance's restrictions on its About page
* `Instance:UnlistedTwitterAccounts` (default: null) to enable unlisted publication for selected twitter accounts, separated by `;` (please limit this to brands and other public profiles).
* `Instance:SensitiveTwitterAccounts` (default: null) mark all media from given accounts as sensitive by default, separated by `;`.
* `Instance:FailingTwitterUserCleanUpThreshold` (default: 700) set the max allowed errors (due to a banned/deleted/private account) from a Twitter Account retrieval before auto-removal. (by default an account is called every 15 mins)
# Docker Compose full example
@ -88,6 +91,7 @@ services:
+ - Instance:TwitterDomainLabel=Nitter
+ - Instance:InfoBanner=This is my BirdsiteLIVE instance. There are many like it, but this one is mine.
+ - Instance:ShowAboutInstanceOnProfiles=true
+ - Instance:SensitiveTwitterAccounts=archillect
networks:
[...]


+ 10
- 2
src/BSLManager/App.cs View File

@ -146,23 +146,31 @@ namespace BSLManager
Width = Dim.Fill(),
Height = 1
};
var inbox = new Label($"Inbox: {follower.InboxRoute}")
var errors = new Label($"Posting Errors: {follower.PostingErrorCount}")
{
X = 1,
Y = 4,
Width = Dim.Fill(),
Height = 1
};
var sharedInbox = new Label($"Shared Inbox: {follower.SharedInboxRoute}")
var inbox = new Label($"Inbox: {follower.InboxRoute}")
{
X = 1,
Y = 5,
Width = Dim.Fill(),
Height = 1
};
var sharedInbox = new Label($"Shared Inbox: {follower.SharedInboxRoute}")
{
X = 1,
Y = 6,
Width = Dim.Fill(),
Height = 1
};
dialog.Add(name);
dialog.Add(following);
dialog.Add(errors);
dialog.Add(inbox);
dialog.Add(sharedInbox);
dialog.Add(close);


+ 1
- 1
src/BSLManager/Domain/FollowersListState.cs View File

@ -26,7 +26,7 @@ namespace BSLManager.Domain
foreach (var follower in _sourceUserList)
{
var displayedUser = $"{GetFullHandle(follower)} ({follower.Followings.Count})";
var displayedUser = $"{GetFullHandle(follower)} ({follower.Followings.Count}) (err:{follower.PostingErrorCount})";
_filteredDisplayableUserList.Add(displayedUser);
}
}


+ 4
- 1
src/BirdsiteLive.Common/Settings/InstanceSettings.cs View File

@ -23,5 +23,8 @@
public bool DiscloseInstanceRestrictions { get; set; }
public string SensitiveTwitterAccounts { get; set; }
public int FailingTwitterUserCleanUpThreshold { get; set; }
}
}
}

+ 11
- 1
src/BirdsiteLive.Domain/Repository/PublicationRepository.cs View File

@ -7,16 +7,19 @@ namespace BirdsiteLive.Domain.Repository
public interface IPublicationRepository
{
bool IsUnlisted(string twitterAcct);
bool IsSensitive(string twitterAcct);
}
public class PublicationRepository : IPublicationRepository
{
private readonly string[] _unlistedAccounts;
private readonly string[] _sensitiveAccounts;
#region Ctor
public PublicationRepository(InstanceSettings settings)
{
_unlistedAccounts = PatternsParser.Parse(settings.UnlistedTwitterAccounts);
_sensitiveAccounts = PatternsParser.Parse(settings.SensitiveTwitterAccounts);
}
#endregion
@ -26,5 +29,12 @@ namespace BirdsiteLive.Domain.Repository
return _unlistedAccounts.Contains(twitterAcct.ToLowerInvariant());
}
public bool IsSensitive(string twitterAcct)
{
if (_sensitiveAccounts == null || !_sensitiveAccounts.Any()) return false;
return _sensitiveAccounts.Contains(twitterAcct.ToLowerInvariant());
}
}
}
}

+ 8
- 2
src/BirdsiteLive.Domain/StatusService.cs View File

@ -50,6 +50,11 @@ namespace BirdsiteLive.Domain
if (isUnlisted)
cc = new[] {"https://www.w3.org/ns/activitystreams#Public"};
string summary = null;
var sensitive = _publicationRepository.IsSensitive(username);
if (sensitive)
summary = "Potential Content Warning";
var extractedTags = _statusExtractor.Extract(tweet.MessageContent);
_statisticsHandler.ExtractedStatus(extractedTags.tags.Count(x => x.type == "Mention"));
@ -81,7 +86,8 @@ namespace BirdsiteLive.Domain
to = new[] { to },
cc = cc,
sensitive = tweet.IsSensitive,
sensitive = tweet.IsSensitive || sensitive,
summary = summary,
content = $"<p>{content}</p>",
attachment = Convert(tweet.Media),
tag = extractedTags.tags,
@ -104,4 +110,4 @@ namespace BirdsiteLive.Domain
}).ToArray();
}
}
}
}

+ 1
- 0
src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj View File

@ -13,6 +13,7 @@
<ItemGroup>
<ProjectReference Include="..\BirdsiteLive.Domain\BirdsiteLive.Domain.csproj" />
<ProjectReference Include="..\BirdsiteLive.Moderation\BirdsiteLive.Moderation.csproj" />
<ProjectReference Include="..\BirdsiteLive.Twitter\BirdsiteLive.Twitter.csproj" />
<ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
</ItemGroup>


+ 12
- 0
src/BirdsiteLive.Pipeline/Contracts/IRefreshTwitterUserStatusProcessor.cs View File

@ -0,0 +1,12 @@
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Models;
namespace BirdsiteLive.Pipeline.Contracts
{
public interface IRefreshTwitterUserStatusProcessor
{
Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct);
}
}

+ 1
- 1
src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs View File

@ -7,7 +7,7 @@ namespace BirdsiteLive.Pipeline.Contracts
{
public interface IRetrieveFollowersProcessor
{
Task<IEnumerable<UserWithTweetsToSync>> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct);
Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct);
//IAsyncEnumerable<UserWithTweetsToSync> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct);
}
}

+ 1
- 1
src/BirdsiteLive.Pipeline/Contracts/IRetrieveTweetsProcessor.cs View File

@ -7,6 +7,6 @@ namespace BirdsiteLive.Pipeline.Contracts
{
public interface IRetrieveTweetsProcessor
{
Task<UserWithTweetsToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct);
Task<UserWithDataToSync[]> ProcessAsync(UserWithDataToSync[] syncTwitterUsers, CancellationToken ct);
}
}

+ 1
- 1
src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs View File

@ -6,6 +6,6 @@ namespace BirdsiteLive.Pipeline.Contracts
{
public interface ISaveProgressionProcessor
{
Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct);
Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
}
}

+ 1
- 1
src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs View File

@ -6,6 +6,6 @@ namespace BirdsiteLive.Pipeline.Contracts
{
public interface ISendTweetsToFollowersProcessor
{
Task<UserWithTweetsToSync> ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct);
Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
}
}

src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs → src/BirdsiteLive.Pipeline/Models/UserWithDataToSync.cs View File


+ 74
- 0
src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs View File

@ -0,0 +1,74 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Moderation.Actions;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Models;
using BirdsiteLive.Twitter;
namespace BirdsiteLive.Pipeline.Processors
{
public class RefreshTwitterUserStatusProcessor : IRefreshTwitterUserStatusProcessor
{
private readonly ICachedTwitterUserService _twitterUserService;
private readonly ITwitterUserDal _twitterUserDal;
private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
private readonly InstanceSettings _instanceSettings;
#region Ctor
public RefreshTwitterUserStatusProcessor(ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, IRemoveTwitterAccountAction removeTwitterAccountAction, InstanceSettings instanceSettings)
{
_twitterUserService = twitterUserService;
_twitterUserDal = twitterUserDal;
_removeTwitterAccountAction = removeTwitterAccountAction;
_instanceSettings = instanceSettings;
}
#endregion
public async Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct)
{
var usersWtData = new List<UserWithDataToSync>();
foreach (var user in syncTwitterUsers)
{
var userView = _twitterUserService.GetUser(user.Acct);
if (userView == null)
{
await AnalyseFailingUserAsync(user);
}
else if (!userView.Protected)
{
user.FetchingErrorCount = 0;
var userWtData = new UserWithDataToSync
{
User = user
};
usersWtData.Add(userWtData);
}
}
return usersWtData.ToArray();
}
private async Task AnalyseFailingUserAsync(SyncTwitterUser user)
{
var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
dbUser.FetchingErrorCount++;
if (dbUser.FetchingErrorCount > _instanceSettings.FailingTwitterUserCleanUpThreshold)
{
await _removeTwitterAccountAction.ProcessAsync(user);
}
else
{
await _twitterUserDal.UpdateTwitterUserAsync(dbUser);
}
// Purge
_twitterUserService.PurgeUser(user.Acct);
}
}
}

+ 1
- 1
src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs View File

@ -18,7 +18,7 @@ namespace BirdsiteLive.Pipeline.Processors
}
#endregion
public async Task<IEnumerable<UserWithTweetsToSync>> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct)
public async Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct)
{
//TODO multithread this
foreach (var user in userWithTweetsToSyncs)


+ 9
- 16
src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs View File

@ -31,33 +31,30 @@ namespace BirdsiteLive.Pipeline.Processors
}
#endregion
public async Task<UserWithTweetsToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct)
public async Task<UserWithDataToSync[]> ProcessAsync(UserWithDataToSync[] syncTwitterUsers, CancellationToken ct)
{
var usersWtTweets = new List<UserWithTweetsToSync>();
var usersWtTweets = new List<UserWithDataToSync>();
//TODO multithread this
foreach (var user in syncTwitterUsers)
foreach (var userWtData in syncTwitterUsers)
{
var user = userWtData.User;
var tweets = RetrieveNewTweets(user);
if (tweets.Length > 0 && user.LastTweetPostedId != -1)
{
var userWtTweets = new UserWithTweetsToSync
{
User = user,
Tweets = tweets
};
usersWtTweets.Add(userWtTweets);
userWtData.Tweets = tweets;
usersWtTweets.Add(userWtData);
}
else if (tweets.Length > 0 && user.LastTweetPostedId == -1)
{
var tweetId = tweets.Last().Id;
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, now);
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now);
}
else
{
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, now);
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
}
}
@ -67,11 +64,7 @@ namespace BirdsiteLive.Pipeline.Processors
private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
{
var tweets = new ExtractedTweet[0];
// Don't retrieve TL if protected
var userView = _twitterUserService.GetUser(user.Acct);
if (userView == null || userView.Protected) return tweets;
try
{
if (user.LastTweetPostedId == -1)


+ 6
- 1
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs View File

@ -55,7 +55,12 @@ namespace BirdsiteLive.Pipeline.Processors
}
var splitCount = splitUsers.Count();
if (splitCount < 15) await Task.Delay((15 - splitCount) * WaitFactor, ct);
if (splitCount < 15) await Task.Delay((15 - splitCount) * WaitFactor, ct); //Always wait 15min
//// Extra wait time to fit 100.000/day limit
//var extraWaitTime = (int)Math.Ceiling((60 / ((100000d / 24) / userCount)) - 15);
//if (extraWaitTime < 0) extraWaitTime = 0;
//await Task.Delay(extraWaitTime * 1000, ct);
}
catch (Exception e)
{


+ 2
- 2
src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs View File

@ -22,7 +22,7 @@ namespace BirdsiteLive.Pipeline.Processors
}
#endregion
public async Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct)
public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
{
try
{
@ -49,7 +49,7 @@ namespace BirdsiteLive.Pipeline.Processors
var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
var minimumSync = followingSyncStatuses.Min();
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, now);
await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now);
}
catch (Exception e)
{


+ 31
- 6
src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs View File

@ -22,18 +22,20 @@ namespace BirdsiteLive.Pipeline.Processors
{
private readonly ISendTweetsToInboxTask _sendTweetsToInboxTask;
private readonly ISendTweetsToSharedInboxTask _sendTweetsToSharedInbox;
private readonly IFollowersDal _followersDal;
private readonly ILogger<SendTweetsToFollowersProcessor> _logger;
#region Ctor
public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, ILogger<SendTweetsToFollowersProcessor> logger)
public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger<SendTweetsToFollowersProcessor> logger)
{
_sendTweetsToInboxTask = sendTweetsToInboxTask;
_sendTweetsToSharedInbox = sendTweetsToSharedInbox;
_logger = logger;
_followersDal = followersDal;
}
#endregion
public async Task<UserWithTweetsToSync> ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct)
public async Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
{
var user = userWithTweetsToSync.User;
@ -41,18 +43,18 @@ namespace BirdsiteLive.Pipeline.Processors
var followersWtSharedInbox = userWithTweetsToSync.Followers
.Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
.ToList();
await ProcessFollowersWithSharedInbox(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
// Process Inbox
var followerWtInbox = userWithTweetsToSync.Followers
.Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
.ToList();
await ProcessFollowersWithInbox(userWithTweetsToSync.Tweets, followerWtInbox, user);
await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
return userWithTweetsToSync;
}
private async Task ProcessFollowersWithSharedInbox(ExtractedTweet[] tweets, List<Follower> followers, SyncTwitterUser user)
private async Task ProcessFollowersWithSharedInboxAsync(ExtractedTweet[] tweets, List<Follower> followers, SyncTwitterUser user)
{
var followersPerInstances = followers.GroupBy(x => x.Host);
@ -61,28 +63,51 @@ namespace BirdsiteLive.Pipeline.Processors
try
{
await _sendTweetsToSharedInbox.ExecuteAsync(tweets, user, followersPerInstance.Key, followersPerInstance.ToArray());
foreach (var f in followersPerInstance)
await ProcessWorkingUserAsync(f);
}
catch (Exception e)
{
var follower = followersPerInstance.First();
_logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.SharedInboxRoute);
foreach (var f in followersPerInstance)
await ProcessFailingUserAsync(f);
}
}
}
private async Task ProcessFollowersWithInbox(ExtractedTweet[] tweets, List<Follower> followerWtInbox, SyncTwitterUser user)
private async Task ProcessFollowersWithInboxAsync(ExtractedTweet[] tweets, List<Follower> followerWtInbox, SyncTwitterUser user)
{
foreach (var follower in followerWtInbox)
{
try
{
await _sendTweetsToInboxTask.ExecuteAsync(tweets, follower, user);
await ProcessWorkingUserAsync(follower);
}
catch (Exception e)
{
_logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.InboxRoute);
await ProcessFailingUserAsync(follower);
}
}
}
private async Task ProcessWorkingUserAsync(Follower follower)
{
if (follower.PostingErrorCount > 0)
{
follower.PostingErrorCount = 0;
await _followersDal.UpdateFollowerAsync(follower);
}
}
private async Task ProcessFailingUserAsync(Follower follower)
{
follower.PostingErrorCount++;
await _followersDal.UpdateFollowerAsync(follower);
}
}
}

+ 18
- 10
src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs View File

@ -1,4 +1,5 @@
using System;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
@ -17,6 +18,7 @@ namespace BirdsiteLive.Pipeline
public class StatusPublicationPipeline : IStatusPublicationPipeline
{
private readonly IRetrieveTwitterUsersProcessor _retrieveTwitterAccountsProcessor;
private readonly IRefreshTwitterUserStatusProcessor _refreshTwitterUserStatusProcessor;
private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor;
private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor;
private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor;
@ -24,13 +26,14 @@ namespace BirdsiteLive.Pipeline
private readonly ILogger<StatusPublicationPipeline> _logger;
#region Ctor
public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionProcessor saveProgressionProcessor, ILogger<StatusPublicationPipeline> logger)
public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionProcessor saveProgressionProcessor, IRefreshTwitterUserStatusProcessor refreshTwitterUserStatusProcessor, ILogger<StatusPublicationPipeline> logger)
{
_retrieveTweetsProcessor = retrieveTweetsProcessor;
_retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
_retrieveFollowersProcessor = retrieveFollowersProcessor;
_sendTweetsToFollowersProcessor = sendTweetsToFollowersProcessor;
_saveProgressionProcessor = saveProgressionProcessor;
_refreshTwitterUserStatusProcessor = refreshTwitterUserStatusProcessor;
_logger = logger;
}
@ -39,16 +42,21 @@ namespace BirdsiteLive.Pipeline
public async Task ExecuteAsync(CancellationToken ct)
{
// Create blocks
var twitterUsersBufferBlock = new BufferBlock<SyncTwitterUser[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
var retrieveTweetsBlock = new TransformBlock<SyncTwitterUser[], UserWithTweetsToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct));
var retrieveTweetsBufferBlock = new BufferBlock<UserWithTweetsToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
var retrieveFollowersBlock = new TransformManyBlock<UserWithTweetsToSync[], UserWithTweetsToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
var retrieveFollowersBufferBlock = new BufferBlock<UserWithTweetsToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
var sendTweetsToFollowersBlock = new TransformBlock<UserWithTweetsToSync, UserWithTweetsToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
var sendTweetsToFollowersBufferBlock = new BufferBlock<UserWithTweetsToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
var saveProgressionBlock = new ActionBlock<UserWithTweetsToSync>(async x => await _saveProgressionProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
var twitterUserToRefreshBufferBlock = new BufferBlock<SyncTwitterUser[]>(new DataflowBlockOptions
{ BoundedCapacity = 1, CancellationToken = ct });
var twitterUserToRefreshBlock = new TransformBlock<SyncTwitterUser[], UserWithDataToSync[]>(async x => await _refreshTwitterUserStatusProcessor.ProcessAsync(x, ct));
var twitterUsersBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct));
var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
var sendTweetsToFollowersBlock = new TransformBlock<UserWithDataToSync, UserWithDataToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
var sendTweetsToFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
var saveProgressionBlock = new ActionBlock<UserWithDataToSync>(async x => await _saveProgressionProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
// Link pipeline
twitterUserToRefreshBufferBlock.LinkTo(twitterUserToRefreshBlock, new DataflowLinkOptions { PropagateCompletion = true });
twitterUserToRefreshBlock.LinkTo(twitterUsersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
@ -58,7 +66,7 @@ namespace BirdsiteLive.Pipeline
sendTweetsToFollowersBufferBlock.LinkTo(saveProgressionBlock, new DataflowLinkOptions { PropagateCompletion = true });
// Launch twitter user retriever
var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUsersBufferBlock, ct);
var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUserToRefreshBufferBlock, ct);
// Wait
await Task.WhenAny(new[] { retrieveTwitterAccountsTask, saveProgressionBlock.Completion });


+ 1
- 1
src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs View File

@ -95,7 +95,7 @@ namespace BirdsiteLive.Statistics.Domain
TimelineCallsCountMin = timelineCalls.Any() ? timelineCalls.Min() : 0,
TimelineCallsCountAvg = timelineCalls.Any() ? (int)timelineCalls.Average() : 0,
TimelineCallsCountMax = timelineCalls.Any() ? timelineCalls.Max() : 0,
TimelineCallsMax = 1500
TimelineCallsMax = 1000
};
}
}


+ 3
- 0
src/BirdsiteLive.Twitter/TwitterUserService.cs View File

@ -49,6 +49,9 @@ namespace BirdsiteLive.Twitter
catch (Exception e)
{
_logger.LogError(e, "Error retrieving user {Username}", username);
// TODO keep track of error, see where to remove user if too much errors
return null;
}


+ 1
- 1
src/BirdsiteLive/BirdsiteLive.csproj View File

@ -4,7 +4,7 @@
<TargetFramework>netcoreapp3.1</TargetFramework>
<UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
<DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
<Version>0.17.0</Version>
<Version>0.19.0</Version>
</PropertyGroup>
<ItemGroup>


+ 3
- 1
src/BirdsiteLive/Controllers/StatisticsController.cs View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Domain.Statistics;
@ -16,7 +17,6 @@ namespace BirdsiteLive.Controllers
private readonly ITwitterStatisticsHandler _twitterStatistics;
private readonly IExtractionStatisticsHandler _extractionStatistics;
#region Ctor
public StatisticsController(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ITwitterStatisticsHandler twitterStatistics, IExtractionStatisticsHandler extractionStatistics)
{
@ -32,7 +32,9 @@ namespace BirdsiteLive.Controllers
var stats = new Models.StatisticsModels.Statistics
{
FollowersCount = await _followersDal.GetFollowersCountAsync(),
FailingFollowersCount = await _followersDal.GetFailingFollowersCountAsync(),
TwitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync(),
FailingTwitterUserCount = await _twitterUserDal.GetFailingTwitterUsersCountAsync(),
TwitterStatistics = _twitterStatistics.GetStatistics(),
ExtractionStatistics = _extractionStatistics.GetStatistics(),
};


+ 1
- 0
src/BirdsiteLive/Controllers/UsersController.cs View File

@ -60,6 +60,7 @@ namespace BirdsiteLive.Controllers
[Route("/@{id}")]
[Route("/users/{id}")]
[Route("/users/{id}/remote_follow")]
public IActionResult Index(string id)
{
_logger.LogTrace("User Index: {Id}", id);


+ 2
- 0
src/BirdsiteLive/Models/StatisticsModels/Statistics.cs View File

@ -6,7 +6,9 @@ namespace BirdsiteLive.Models.StatisticsModels
public class Statistics
{
public int FollowersCount { get; set; }
public int FailingFollowersCount { get; set; }
public int TwitterUserCount { get; set; }
public int FailingTwitterUserCount { get; set; }
public ApiStatistics TwitterStatistics { get; set; }
public ExtractionStatistics ExtractionStatistics { get; set; }
}

+ 2
- 0
src/BirdsiteLive/Views/Statistics/Index.cshtml View File

@ -9,7 +9,9 @@
<h4>Instance</h4>
<ul>
<li>Twitter Users: @Model.TwitterUserCount</li>
<li>Failing Twitter Users: @Model.FailingTwitterUserCount</li>
<li>Followers: @Model.FollowersCount</li>
<li>Failing Followers: @Model.FailingFollowersCount</li>
</ul>
<h4>Twitter API (Min, Avg, Max for the last 24h)</h4>


+ 3
- 1
src/BirdsiteLive/appsettings.json View File

@ -27,7 +27,9 @@
"InfoBanner": "",
"ShowAboutInstanceOnProfiles": true,
"MaxFollowsPerUser": 0,
"DiscloseInstanceRestrictions": false
"DiscloseInstanceRestrictions": false,
"SensitiveTwitterAccounts": null,
"FailingTwitterUserCleanUpThreshold": 700
},
"Db": {
"Type": "postgres",


+ 14
- 2
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs View File

@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal
{
private readonly PostgresTools _tools;
private readonly Version _currentVersion = new Version(2, 1);
private readonly Version _currentVersion = new Version(2, 3);
private const string DbVersionType = "db-version";
#region Ctor
@ -132,7 +132,9 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
return new[]
{
new Tuple<Version, Version>(new Version(1,0), new Version(2,0)),
new Tuple<Version, Version>(new Version(2,0), new Version(2,1))
new Tuple<Version, Version>(new Version(2,0), new Version(2,1)),
new Tuple<Version, Version>(new Version(2,1), new Version(2,2)),
new Tuple<Version, Version>(new Version(2,2), new Version(2,3))
};
}
@ -151,6 +153,16 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
var addActorId = $@"ALTER TABLE {_settings.FollowersTableName} ADD actorId VARCHAR(2048)";
await _tools.ExecuteRequestAsync(addActorId);
}
else if (from == new Version(2, 1) && to == new Version(2, 2))
{
var addLastSync = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD fetchingErrorCount SMALLINT";
await _tools.ExecuteRequestAsync(addLastSync);
}
else if (from == new Version(2, 2) && to == new Version(2, 3))
{
var addPostingError = $@"ALTER TABLE {_settings.FollowersTableName} ADD postingErrorCount SMALLINT";
await _tools.ExecuteRequestAsync(addPostingError);
}
else
{
throw new NotImplementedException();


+ 18
- 3
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs View File

@ -53,6 +53,19 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
}
}
public async Task<int> GetFailingFollowersCountAsync()
{
var query = $"SELECT COUNT(*) FROM {_settings.FollowersTableName} WHERE postingErrorCount > 0";
using (var dbConnection = Connection)
{
dbConnection.Open();
var result = (await dbConnection.QueryAsync<int>(query)).FirstOrDefault();
return result;
}
}
public async Task<Follower> GetFollowerAsync(string acct, string host)
{
var query = $"SELECT * FROM {_settings.FollowersTableName} WHERE acct = @acct AND host = @host";
@ -103,13 +116,13 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
if (follower.Id == default) throw new ArgumentException("id");
var serializedDic = JsonConvert.SerializeObject(follower.FollowingsSyncStatus);
var query = $"UPDATE {_settings.FollowersTableName} SET followings = @followings, followingsSyncStatus = CAST(@followingsSyncStatus as json) WHERE id = @id";
var query = $"UPDATE {_settings.FollowersTableName} SET followings = @followings, followingsSyncStatus = CAST(@followingsSyncStatus as json), postingErrorCount = @postingErrorCount WHERE id = @id";
using (var dbConnection = Connection)
{
dbConnection.Open();
await dbConnection.QueryAsync(query, new { follower.Id, follower.Followings, followingsSyncStatus = serializedDic });
await dbConnection.QueryAsync(query, new { follower.Id, follower.Followings, followingsSyncStatus = serializedDic, postingErrorCount = follower.PostingErrorCount });
}
}
@ -158,7 +171,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
ActorId = follower.ActorId,
SharedInboxRoute = follower.SharedInboxRoute,
Followings = follower.Followings.ToList(),
FollowingsSyncStatus = JsonConvert.DeserializeObject<Dictionary<int,long>>(follower.FollowingsSyncStatus)
FollowingsSyncStatus = JsonConvert.DeserializeObject<Dictionary<int,long>>(follower.FollowingsSyncStatus),
PostingErrorCount = follower.PostingErrorCount
};
}
}
@ -174,5 +188,6 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
public string InboxRoute { get; set; }
public string SharedInboxRoute { get; set; }
public string ActorId { get; set; }
public int PostingErrorCount { get; set; }
}
}

+ 21
- 3
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs View File

@ -73,6 +73,19 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
}
}
public async Task<int> GetFailingTwitterUsersCountAsync()
{
var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0";
using (var dbConnection = Connection)
{
dbConnection.Open();
var result = (await dbConnection.QueryAsync<int>(query)).FirstOrDefault();
return result;
}
}
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber)
{
var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC LIMIT @maxNumber";
@ -99,23 +112,28 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
}
}
public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync)
public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync)
{
if(id == default) throw new ArgumentException("id");
if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId");
if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId");
if(lastSync == default) throw new ArgumentException("lastSync");
var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, lastSync = @lastSync WHERE id = @id";
var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync WHERE id = @id";
using (var dbConnection = Connection)
{
dbConnection.Open();
await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId, lastSync = lastSync.ToUniversalTime() });
await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId, fetchingErrorCount, lastSync = lastSync.ToUniversalTime() });
}
}
public async Task UpdateTwitterUserAsync(SyncTwitterUser user)
{
await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync);
}
public async Task DeleteTwitterUserAsync(string acct)
{
if (string.IsNullOrWhiteSpace(acct)) throw new ArgumentException("acct");


+ 1
- 0
src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs View File

@ -15,5 +15,6 @@ namespace BirdsiteLive.DAL.Contracts
Task DeleteFollowerAsync(int id);
Task DeleteFollowerAsync(string acct, string host);
Task<int> GetFollowersCountAsync();
Task<int> GetFailingFollowersCountAsync();
}
}

+ 3
- 1
src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs View File

@ -11,9 +11,11 @@ namespace BirdsiteLive.DAL.Contracts
Task<SyncTwitterUser> GetTwitterUserAsync(int id);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync();
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync);
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync);
Task UpdateTwitterUserAsync(SyncTwitterUser user);
Task DeleteTwitterUserAsync(string acct);
Task DeleteTwitterUserAsync(int id);
Task<int> GetTwitterUsersCountAsync();
Task<int> GetFailingTwitterUsersCountAsync();
}
}

+ 2
- 0
src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs View File

@ -14,5 +14,7 @@ namespace BirdsiteLive.DAL.Models
public string Host { get; set; }
public string InboxRoute { get; set; }
public string SharedInboxRoute { get; set; }
public int PostingErrorCount { get; set; }
}
}

+ 2
- 0
src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs View File

@ -11,5 +11,7 @@ namespace BirdsiteLive.DAL.Models
public long LastTweetSynchronizedForAllFollowersId { get; set; }
public DateTime LastSync { get; set; }
public int FetchingErrorCount { get; set; } //TODO: update DAL
}
}

+ 91
- 3
src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs View File

@ -54,6 +54,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.AreEqual(inboxRoute, result.InboxRoute);
Assert.AreEqual(sharedInboxRoute, result.SharedInboxRoute);
Assert.AreEqual(actorId, result.ActorId);
Assert.AreEqual(0, result.PostingErrorCount);
Assert.AreEqual(following.Length, result.Followings.Count);
Assert.AreEqual(following[0], result.Followings[0]);
Assert.AreEqual(followingSync.Count, result.FollowingsSyncStatus.Count);
@ -83,6 +84,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.AreEqual(sharedInboxRoute, result.SharedInboxRoute);
Assert.AreEqual(0, result.Followings.Count);
Assert.AreEqual(0, result.FollowingsSyncStatus.Count);
Assert.AreEqual(0, result.PostingErrorCount);
}
[TestMethod]
@ -125,6 +127,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.AreEqual(followingSync.Count, result.FollowingsSyncStatus.Count);
Assert.AreEqual(followingSync.First().Key, result.FollowingsSyncStatus.First().Key);
Assert.AreEqual(followingSync.First().Value, result.FollowingsSyncStatus.First().Value);
Assert.AreEqual(0, result.PostingErrorCount);
}
[TestMethod]
@ -234,7 +237,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
actorId = $"https://{host}/{acct}";
await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
//User 2
//User 3
acct = "myhandle3";
host = "domain.ext";
following = new[] { 1 };
@ -247,6 +250,54 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.AreEqual(3, result);
}
[TestMethod]
public async Task CountFailingFollowersAsync()
{
var dal = new FollowersPostgresDal(_settings);
var result = await dal.GetFailingFollowersCountAsync();
Assert.AreEqual(0, result);
//User 1
var acct = "myhandle1";
var host = "domain.ext";
var following = new[] { 1, 2, 3 };
var followingSync = new Dictionary<int, long>();
var inboxRoute = "/myhandle1/inbox";
var sharedInboxRoute = "/inbox";
var actorId = $"https://{host}/{acct}";
await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
//User 2
acct = "myhandle2";
host = "domain.ext";
following = new[] { 2, 4, 5 };
inboxRoute = "/myhandle2/inbox";
sharedInboxRoute = "/inbox2";
actorId = $"https://{host}/{acct}";
await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
var follower = await dal.GetFollowerAsync(acct, host);
follower.PostingErrorCount = 1;
await dal.UpdateFollowerAsync(follower);
//User 3
acct = "myhandle3";
host = "domain.ext";
following = new[] { 1 };
inboxRoute = "/myhandle3/inbox";
sharedInboxRoute = "/inbox3";
actorId = $"https://{host}/{acct}";
await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
follower = await dal.GetFollowerAsync(acct, host);
follower.PostingErrorCount = 50;
await dal.UpdateFollowerAsync(follower);
result = await dal.GetFailingFollowersCountAsync();
Assert.AreEqual(2, result);
}
[TestMethod]
public async Task CreateUpdateAndGetFollower_Add()
{
@ -276,8 +327,8 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
};
result.Followings = updatedFollowing.ToList();
result.FollowingsSyncStatus = updatedFollowingSync;
result.PostingErrorCount = 10;
await dal.UpdateFollowerAsync(result);
result = await dal.GetFollowerAsync(acct, host);
@ -286,6 +337,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.AreEqual(updatedFollowingSync.Count, result.FollowingsSyncStatus.Count);
Assert.AreEqual(updatedFollowingSync.First().Key, result.FollowingsSyncStatus.First().Key);
Assert.AreEqual(updatedFollowingSync.First().Value, result.FollowingsSyncStatus.First().Value);
Assert.AreEqual(10, result.PostingErrorCount);
}
[TestMethod]
@ -316,6 +368,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
};
result.Followings = updatedFollowing.ToList();
result.FollowingsSyncStatus = updatedFollowingSync;
result.PostingErrorCount = 5;
await dal.UpdateFollowerAsync(result);
result = await dal.GetFollowerAsync(acct, host);
@ -325,6 +378,41 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.AreEqual(updatedFollowingSync.Count, result.FollowingsSyncStatus.Count);
Assert.AreEqual(updatedFollowingSync.First().Key, result.FollowingsSyncStatus.First().Key);
Assert.AreEqual(updatedFollowingSync.First().Value, result.FollowingsSyncStatus.First().Value);
Assert.AreEqual(5, result.PostingErrorCount);
}
[TestMethod]
public async Task CreateUpdateAndGetFollower_ResetErrorCount()
{
var acct = "myhandle";
var host = "domain.ext";
var following = new[] { 12, 19, 23 };
var followingSync = new Dictionary<int, long>()
{
{12, 165L},
{19, 166L},
{23, 167L}
};
var inboxRoute = "/myhandle/inbox";
var sharedInboxRoute = "/inbox";
var actorId = $"https://{host}/{acct}";
var dal = new FollowersPostgresDal(_settings);
await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
var result = await dal.GetFollowerAsync(acct, host);
Assert.AreEqual(0, result.PostingErrorCount);
result.PostingErrorCount = 5;
await dal.UpdateFollowerAsync(result);
result = await dal.GetFollowerAsync(acct, host);
Assert.AreEqual(5, result.PostingErrorCount);
result.PostingErrorCount = 0;
await dal.UpdateFollowerAsync(result);
result = await dal.GetFollowerAsync(acct, host);
Assert.AreEqual(0, result.PostingErrorCount);
}
[TestMethod]


+ 64
- 6
src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs View File

@ -1,4 +1,5 @@
using System;
using System.Diagnostics;
using System.Threading.Tasks;
using System.Xml;
using BirdsiteLive.DAL.Postgres.DataAccessLayers;
@ -47,6 +48,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.AreEqual(acct, result.Acct);
Assert.AreEqual(lastTweetId, result.LastTweetPostedId);
Assert.AreEqual(lastTweetId, result.LastTweetSynchronizedForAllFollowersId);
Assert.AreEqual(0, result.FetchingErrorCount);
Assert.IsTrue(result.Id > 0);
}
@ -83,13 +85,47 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
var updatedLastTweetId = 1600L;
var updatedLastSyncId = 1550L;
var now = DateTime.Now;
await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, now);
var errors = 15;
await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now);
result = await dal.GetTwitterUserAsync(acct);
Assert.AreEqual(acct, result.Acct);
Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
Assert.AreEqual(errors, result.FetchingErrorCount);
Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
}
[TestMethod]
public async Task CreateUpdate2AndGetUser()
{
var acct = "myid";
var lastTweetId = 1548L;
var dal = new TwitterUserPostgresDal(_settings);
await dal.CreateTwitterUserAsync(acct, lastTweetId);
var result = await dal.GetTwitterUserAsync(acct);
var updatedLastTweetId = 1600L;
var updatedLastSyncId = 1550L;
var now = DateTime.Now;
var errors = 15;
result.LastTweetPostedId = updatedLastTweetId;
result.LastTweetSynchronizedForAllFollowersId = updatedLastSyncId;
result.FetchingErrorCount = errors;
result.LastSync = now;
await dal.UpdateTwitterUserAsync(result);
result = await dal.GetTwitterUserAsync(acct);
Assert.AreEqual(acct, result.Acct);
Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
Assert.AreEqual(errors, result.FetchingErrorCount);
Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
}
@ -98,7 +134,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
public async Task Update_NoId()
{
var dal = new TwitterUserPostgresDal(_settings);
await dal.UpdateTwitterUserAsync(default, default, default, DateTime.UtcNow);
await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow);
}
[TestMethod]
@ -106,7 +142,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
public async Task Update_NoLastTweetPostedId()
{
var dal = new TwitterUserPostgresDal(_settings);
await dal.UpdateTwitterUserAsync(12, default, default, DateTime.UtcNow);
await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow);
}
[TestMethod]
@ -114,7 +150,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
public async Task Update_NoLastTweetSynchronizedForAllFollowersId()
{
var dal = new TwitterUserPostgresDal(_settings);
await dal.UpdateTwitterUserAsync(12, 9556, default, DateTime.UtcNow);
await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow);
}
[TestMethod]
@ -122,7 +158,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
public async Task Update_NoLastSync()
{
var dal = new TwitterUserPostgresDal(_settings);
await dal.UpdateTwitterUserAsync(12, 9556, 65, default);
await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default);
}
[TestMethod]
@ -216,7 +252,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
{
var user = allUsers[i];
var date = i % 2 == 0 ? oldest : newest;
await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, date);
await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date);
}
var result = await dal.GetAllTwitterUsersAsync(10);
@ -265,5 +301,27 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
var result = await dal.GetTwitterUsersCountAsync();
Assert.AreEqual(10, result);
}
[TestMethod]
public async Task CountFailingTwitterUsers()
{
var dal = new TwitterUserPostgresDal(_settings);
for (var i = 0; i < 10; i++)
{
var acct = $"myid{i}";
var lastTweetId = 1548L;
await dal.CreateTwitterUserAsync(acct, lastTweetId);
if (i == 0 || i == 2 || i == 3)
{
var t = await dal.GetTwitterUserAsync(acct);
await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now);
}
}
var result = await dal.GetFailingTwitterUsersCountAsync();
Assert.AreEqual(3, result);
}
}
}

+ 333
- 0
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs View File

@ -0,0 +1,333 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Moderation.Actions;
using BirdsiteLive.Pipeline.Models;
using BirdsiteLive.Pipeline.Processors;
using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Models;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
namespace BirdsiteLive.Pipeline.Tests.Processors
{
[TestClass]
public class RefreshTwitterUserStatusProcessorTests
{
[TestMethod]
public async Task ProcessAsync_Test()
{
#region Stubs
var userId1 = 1;
var userId2 = 2;
var users = new List<SyncTwitterUser>
{
new SyncTwitterUser
{
Id = userId1
},
new SyncTwitterUser
{
Id = userId2
}
};
var settings = new InstanceSettings
{
FailingTwitterUserCleanUpThreshold = 300
};
#endregion
#region Mocks
var twitterUserServiceMock = new Mock<ICachedTwitterUserService>(MockBehavior.Strict);
twitterUserServiceMock
.Setup(x => x.GetUser(It.IsAny<string>()))
.Returns(new TwitterUser
{
Protected = false
});
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
var removeTwitterAccountActionMock = new Mock<IRemoveTwitterAccountAction>(MockBehavior.Strict);
#endregion
var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings);
var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None);
#region Validations
Assert.AreEqual(2 , result.Length);
Assert.IsTrue(result.Any(x => x.User.Id == userId1));
Assert.IsTrue(result.Any(x => x.User.Id == userId2));
twitterUserServiceMock.VerifyAll();
twitterUserDalMock.VerifyAll();
removeTwitterAccountActionMock.VerifyAll();
#endregion
}
[TestMethod]
public async Task ProcessAsync_ResetErrorCount_Test()
{
#region Stubs
var userId1 = 1;
var users = new List<SyncTwitterUser>
{
new SyncTwitterUser
{
Id = userId1,
FetchingErrorCount = 100
}
};
var settings = new InstanceSettings
{
FailingTwitterUserCleanUpThreshold = 300
};
#endregion
#region Mocks
var twitterUserServiceMock = new Mock<ICachedTwitterUserService>(MockBehavior.Strict);
twitterUserServiceMock
.Setup(x => x.GetUser(It.IsAny<string>()))
.Returns(new TwitterUser
{
Protected = false
});
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
var removeTwitterAccountActionMock = new Mock<IRemoveTwitterAccountAction>(MockBehavior.Strict);
#endregion
var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings);
var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None);
#region Validations
Assert.AreEqual(1, result.Length);
Assert.IsTrue(result.Any(x => x.User.Id == userId1));
Assert.AreEqual(0, result.First().User.FetchingErrorCount);
twitterUserServiceMock.VerifyAll();
twitterUserDalMock.VerifyAll();
removeTwitterAccountActionMock.VerifyAll();
#endregion
}
[TestMethod]
public async Task ProcessAsync_Unfound_Test()
{
#region Stubs
var userId1 = 1;
var acct1 = "user1";
var userId2 = 2;
var acct2 = "user2";
var users = new List<SyncTwitterUser>
{
new SyncTwitterUser
{
Id = userId1,
Acct = acct1
},
new SyncTwitterUser
{
Id = userId2,
Acct = acct2
}
};
var settings = new InstanceSettings
{
FailingTwitterUserCleanUpThreshold = 300
};
#endregion
#region Mocks
var twitterUserServiceMock = new Mock<ICachedTwitterUserService>(MockBehavior.Strict);
twitterUserServiceMock
.Setup(x => x.GetUser(It.Is<string>(y => y == acct1)))
.Returns(new TwitterUser
{
Protected = false
});
twitterUserServiceMock
.Setup(x => x.GetUser(It.Is<string>(y => y == acct2)))
.Returns((TwitterUser) null);
twitterUserServiceMock
.Setup(x => x.PurgeUser(It.Is<string>(y => y == acct2)));
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetTwitterUserAsync(It.Is<string>(y => y == acct2)))
.ReturnsAsync(new SyncTwitterUser
{
Id = userId2,
FetchingErrorCount = 0
});
twitterUserDalMock
.Setup(x => x.UpdateTwitterUserAsync(It.Is<SyncTwitterUser>(y => y.Id == userId2 && y.FetchingErrorCount == 1)))
.Returns(Task.CompletedTask);
var removeTwitterAccountActionMock = new Mock<IRemoveTwitterAccountAction>(MockBehavior.Strict);
#endregion
var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings);
var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None);
#region Validations
Assert.AreEqual(1, result.Length);
Assert.IsTrue(result.Any(x => x.User.Id == userId1));
twitterUserServiceMock.VerifyAll();
twitterUserDalMock.VerifyAll();
removeTwitterAccountActionMock.VerifyAll();
#endregion
}
[TestMethod]
public async Task ProcessAsync_Unfound_OverThreshold_Test()
{
#region Stubs
var userId1 = 1;
var acct1 = "user1";
var userId2 = 2;
var acct2 = "user2";
var users = new List<SyncTwitterUser>
{
new SyncTwitterUser
{
Id = userId1,
Acct = acct1
},
new