Merge pull request #20 from NicolasConstant/develop

Develop

.github/workflows/dotnet-core.yml

@@ -1,10 +1,6 @@
-name: .NET Core
+name: ASP.NET Core Build & Tests
 
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
+on: [push, pull_request]  
 
 jobs:
   build:

docker-compose.yml

@@ -0,0 +1,39 @@
+version: "3"
+
+networks:
+    birdsitelivenetwork:
+        external: false
+
+services:
+    server:
+        image: nicolasconstant/birdsitelive:latest
+        restart: always
+        container_name: birdsitelive
+        environment:
+            - Instance:Domain=domain.name
+            - Instance:AdminEmail=name@domain.ext
+            - Db:Type=postgres
+            - Db:Host=db
+            - Db:Name=birdsitelive
+            - Db:User=birdsitelive
+            - Db:Password=birdsitelive
+            - Twitter:ConsumerKey=twitter.api.key
+            - Twitter:ConsumerSecret=twitter.api.key
+        networks:
+            - birdsitelivenetwork
+        ports:
+            - "5000:80"
+        depends_on:
+            - db
+
+    db:
+        image: postgres:9.6
+        restart: always
+        environment:
+            - POSTGRES_USER=birdsitelive
+            - POSTGRES_PASSWORD=birdsitelive
+            - POSTGRES_DB=birdsitelive
+        networks:
+            - birdsitelivenetwork
+        volumes:
+            - ./postgres:/var/lib/postgresql/data

src/BirdsiteLive.ActivityPub/ApDeserializer.cs

@@ -1,4 +1,5 @@
-using Newtonsoft.Json;
+using System;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
@@ -6,41 +7,48 @@ namespace BirdsiteLive.ActivityPub
     {
         public static Activity ProcessActivity(string json)
         {
-            var activity = JsonConvert.DeserializeObject<Activity>(json);
-            switch (activity.type)
+            try
             {
-                case "Follow":
-                    return JsonConvert.DeserializeObject<ActivityFollow>(json);
-                case "Undo":
-                    var a = JsonConvert.DeserializeObject<ActivityUndo>(json);
-                    if(a.apObject.type == "Follow")
-                        return JsonConvert.DeserializeObject<ActivityUndoFollow>(json);
-                    break;
-                case "Accept":
-                    var accept = JsonConvert.DeserializeObject<ActivityAccept>(json);
-                    //var acceptType = JsonConvert.DeserializeObject<Activity>(accept.apObject);
-                    switch ((accept.apObject as dynamic).type.ToString())
-                    {
-                        case "Follow":
-                            var acceptFollow = new ActivityAcceptFollow()
-                            {
-                                type = accept.type,
-                                id = accept.id,
-                                actor = accept.actor,
-                                context = accept.context,
-                                apObject = new ActivityFollow()
+                var activity = JsonConvert.DeserializeObject<Activity>(json);
+                switch (activity.type)
+                {
+                    case "Follow":
+                        return JsonConvert.DeserializeObject<ActivityFollow>(json);
+                    case "Undo":
+                        var a = JsonConvert.DeserializeObject<ActivityUndo>(json);
+                        if(a.apObject.type == "Follow")
+                            return JsonConvert.DeserializeObject<ActivityUndoFollow>(json);
+                        break;
+                    case "Accept":
+                        var accept = JsonConvert.DeserializeObject<ActivityAccept>(json);
+                        //var acceptType = JsonConvert.DeserializeObject<Activity>(accept.apObject);
+                        switch ((accept.apObject as dynamic).type.ToString())
+                        {
+                            case "Follow":
+                                var acceptFollow = new ActivityAcceptFollow()
                                 {
-                                    id = (accept.apObject as dynamic).id?.ToString(),
-                                    type = (accept.apObject as dynamic).type?.ToString(),
-                                    actor = (accept.apObject as dynamic).actor?.ToString(),
-                                    context = (accept.apObject as dynamic).context?.ToString(),
-                                    apObject = (accept.apObject as dynamic).@object?.ToString()
-                                }
-                            };
-                            return acceptFollow;
-                            break;
-                    }
-                    break;
+                                    type = accept.type,
+                                    id = accept.id,
+                                    actor = accept.actor,
+                                    context = accept.context,
+                                    apObject = new ActivityFollow()
+                                    {
+                                        id = (accept.apObject as dynamic).id?.ToString(),
+                                        type = (accept.apObject as dynamic).type?.ToString(),
+                                        actor = (accept.apObject as dynamic).actor?.ToString(),
+                                        context = (accept.apObject as dynamic).context?.ToString(),
+                                        apObject = (accept.apObject as dynamic).@object?.ToString()
+                                    }
+                                };
+                                return acceptFollow;
+                                break;
+                        }
+                        break;
+                }
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine(e);
             }
 
             return null;

src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj

@@ -6,7 +6,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.2" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
     <PackageReference Include="System.Text.Json" Version="4.7.2" />
   </ItemGroup>
 

src/BirdsiteLive.ActivityPub/Models/ActivityAcceptUndoFollow.cs

@@ -0,0 +1,10 @@
+using Newtonsoft.Json;
+
+namespace BirdsiteLive.ActivityPub
+{
+    public class ActivityAcceptUndoFollow : Activity
+    {
+        [JsonProperty("object")]
+        public ActivityUndoFollow apObject { get; set; }
+    }
+}

src/BirdsiteLive.ActivityPub/Models/ActivityCreateNote.cs

@@ -1,4 +1,5 @@
 using System;
+using BirdsiteLive.ActivityPub.Models;
 using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub

src/BirdsiteLive.ActivityPub/Models/Actor.cs

@@ -1,4 +1,5 @@
-using BirdsiteLive.ActivityPub.Converters;
+using System.Net;
+using BirdsiteLive.ActivityPub.Converters;
 using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
@@ -11,13 +12,16 @@ namespace BirdsiteLive.ActivityPub
         public string[] context { get; set; } = new[] { "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1" };
         public string id { get; set; }
         public string type { get; set; }
+        public string followers { get; set; }
         public string preferredUsername { get; set; }
         public string name { get; set; }
         public string summary { get; set; }
         public string url { get; set; }
         public string inbox { get; set; }
+        public bool? discoverable { get; set; } = true;
         public PublicKey publicKey { get; set; }
         public Image icon { get; set; }
         public Image image { get; set; }
+        public EndPoints endpoints { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/Attachment.cs

@@ -0,0 +1,9 @@
+namespace BirdsiteLive.ActivityPub
+{
+    public class Attachment
+    {
+        public string type { get; set; }
+        public string mediaType { get; set; }
+        public string url { get; set; }
+    }
+}

src/BirdsiteLive.ActivityPub/Models/EndPoints.cs

@@ -0,0 +1,7 @@
+namespace BirdsiteLive.ActivityPub
+{
+    public class EndPoints
+    {
+        public string sharedInbox { get; set; }
+    }
+}

src/BirdsiteLive.ActivityPub/Models/Followers.cs

@@ -0,0 +1,15 @@
+using BirdsiteLive.ActivityPub.Converters;
+using Newtonsoft.Json;
+
+namespace BirdsiteLive.ActivityPub.Models
+{
+    public class Followers
+    {
+        [JsonProperty("@context")]
+        [JsonConverter(typeof(ContextArrayConverter))]
+        public string context { get; set; } = "https://www.w3.org/ns/activitystreams";
+
+        public string id { get; set; }
+        public string type { get; set; } = "OrderedCollection";
+    }
+}

src/BirdsiteLive.ActivityPub/Models/Note.cs

@@ -1,9 +1,7 @@
-using System;
-using System.Collections.Generic;
-using BirdsiteLive.ActivityPub.Converters;
+using BirdsiteLive.ActivityPub.Converters;
 using Newtonsoft.Json;
 
-namespace BirdsiteLive.ActivityPub
+namespace BirdsiteLive.ActivityPub.Models
 {
     public class Note
     {
@@ -24,8 +22,8 @@ namespace BirdsiteLive.ActivityPub
         //public string conversation { get; set; }
         public string content { get; set; }
         //public Dictionary<string,string> contentMap { get; set; }
-        public string[] attachment { get; set; }
-        public string[] tag { get; set; }
+        public Attachment[] attachment { get; set; }
+        public Tag[] tag { get; set; }
         //public Dictionary<string, string> replies;
     }
 }

src/BirdsiteLive.ActivityPub/Models/Tag.cs

@@ -0,0 +1,8 @@
+namespace BirdsiteLive.ActivityPub.Models
+{
+    public class Tag {
+        public string type { get; set; } //Hashtag
+        public string href { get; set; } //https://mastodon.social/tags/app
+        public string name { get; set; } //#app
+    }
+}

src/BirdsiteLive.Common/Settings/DbSettings.cs

@@ -0,0 +1,11 @@
+namespace BirdsiteLive.Common.Settings
+{
+    public class DbSettings
+    {
+        public string Type { get; set; }
+        public string Host { get; set; }
+        public string Name { get; set; }
+        public string User { get; set; }
+        public string Password { get; set; }
+    }
+}

src/BirdsiteLive.Common/Settings/InstanceSettings.cs

@@ -3,5 +3,6 @@
     public class InstanceSettings
     {
         public string Domain { get; set; }
+        public string AdminEmail { get; set; }
     }
 }

src/BirdsiteLive.Common/Settings/TwitterSettings.cs

@@ -4,7 +4,5 @@
     {
         public string ConsumerKey { get; set; }
         public string ConsumerSecret { get; set; }
-        public string AccessToken { get; set; }
-        public string AccessTokenSecret { get; set; }
     }
 }

src/BirdsiteLive.Common/Structs/DbTypes.cs

@@ -0,0 +1,7 @@
+namespace BirdsiteLive.Common.Structs
+{
+    public struct DbTypes
+    {
+        public static string Postgres = "postgres";
+    }
+}

src/BirdsiteLive.Domain/ActivityPubService.cs

@@ -1,9 +1,13 @@
 using System;
+using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Security.Cryptography;
 using System.Text;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
+using BirdsiteLive.ActivityPub.Models;
+using BirdsiteLive.Common.Settings;
 using Newtonsoft.Json;
 using Org.BouncyCastle.Bcpg;
 
@@ -13,16 +17,20 @@ namespace BirdsiteLive.Domain
     {
         Task<Actor> GetUser(string objectId);
         Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null);
+        Task<HttpStatusCode> PostNewNoteActivity(Note note, string username, string noteId, string targetHost,
+            string targetInbox);
     }
 
     public class ActivityPubService : IActivityPubService
     {
+        private readonly InstanceSettings _instanceSettings;
         private readonly ICryptoService _cryptoService;
 
         #region Ctor
-        public ActivityPubService(ICryptoService cryptoService)
+        public ActivityPubService(ICryptoService cryptoService, InstanceSettings instanceSettings)
         {
             _cryptoService = cryptoService;
+            _instanceSettings = instanceSettings;
         }
         #endregion
 
@@ -37,6 +45,40 @@ namespace BirdsiteLive.Domain
             }
         }
 
+        public async Task<HttpStatusCode> PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox)
+        {
+            //var username = "gra";
+            var actor = $"https://{_instanceSettings.Domain}/users/{username}";
+            //var targetHost = "mastodon.technology";
+            //var target = $"{targetHost}/users/testtest";
+            //var inbox = $"/users/testtest/inbox";
+
+            //var noteGuid = Guid.NewGuid();
+            var noteUri = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{noteId}";
+            
+            //var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{noteId}";
+            //var to = $"{actor}/followers";
+            //var apPublic = "https://www.w3.org/ns/activitystreams#Public";
+
+            var now = DateTime.UtcNow;
+            var nowString = now.ToString("s") + "Z";
+            
+            var noteActivity = new ActivityCreateNote()
+            {
+                context = "https://www.w3.org/ns/activitystreams",
+                id = $"{noteUri}/activity",
+                type = "Create",
+                actor = actor,
+                published = nowString,
+
+                to = note.to,
+                cc = note.cc,
+                apObject = note
+            };
+
+            return await PostDataAsync(noteActivity, targetHost, actor, targetInbox);
+        }
+
         public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
         {
             var usedInbox = $"/inbox";
@@ -47,20 +89,22 @@ namespace BirdsiteLive.Domain
 
             var date = DateTime.UtcNow.ToUniversalTime();
             var httpDate = date.ToString("r");
-            var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, usedInbox);
 
-            
+            var digest = _cryptoService.ComputeSha256Hash(json);
+
+            var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox);
 
             var client = new HttpClient();
             var httpRequestMessage = new HttpRequestMessage
             {
                 Method = HttpMethod.Post,
-                RequestUri = new Uri($"https://{targetHost}/{usedInbox}"),
+                RequestUri = new Uri($"https://{targetHost}{usedInbox}"),
                 Headers =
                 {
                     {"Host", targetHost},
                     {"Date", httpDate},
-                    {"Signature", signature}
+                    {"Signature", signature},
+                    {"Digest", $"SHA-256={digest}"}
                 },
                 Content = new StringContent(json, Encoding.UTF8, "application/ld+json")
             };
@@ -68,5 +112,7 @@ namespace BirdsiteLive.Domain
             var response = await client.SendAsync(httpRequestMessage);
             return response.StatusCode;
         }
+
+       
     }
 }

src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj

@@ -8,6 +8,7 @@
     <ProjectReference Include="..\BirdsiteLive.ActivityPub\BirdsiteLive.ActivityPub.csproj" />
     <ProjectReference Include="..\BirdsiteLive.Cryptography\BirdsiteLive.Cryptography.csproj" />
     <ProjectReference Include="..\BirdsiteLive.Twitter\BirdsiteLive.Twitter.csproj" />
+    <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
   </ItemGroup>
 
 </Project>

src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs

@@ -0,0 +1,53 @@
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+
+namespace BirdsiteLive.Domain.BusinessUseCases
+{
+    public interface IProcessFollowUser
+    {
+        Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername, string followerInbox, string sharedInbox);
+    }
+
+    public class ProcessFollowUser : IProcessFollowUser
+    {
+        private readonly IFollowersDal _followerDal;
+        private readonly ITwitterUserDal _twitterUserDal;
+
+        #region Ctor
+        public ProcessFollowUser(IFollowersDal followerDal, ITwitterUserDal twitterUserDal)
+        {
+            _followerDal = followerDal;
+            _twitterUserDal = twitterUserDal;
+        }
+        #endregion
+
+        public async Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername, string followerInbox, string sharedInbox)
+        {
+            // Get Follower and Twitter Users
+            var follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain);
+            if (follower == null)
+            {
+                await _followerDal.CreateFollowerAsync(followerUsername, followerDomain, followerInbox, sharedInbox);
+                follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain);
+            }
+
+            var twitterUser = await _twitterUserDal.GetTwitterUserAsync(twitterUsername);
+            if (twitterUser == null)
+            {
+                await _twitterUserDal.CreateTwitterUserAsync(twitterUsername, -1);
+                twitterUser = await _twitterUserDal.GetTwitterUserAsync(twitterUsername);
+            }
+
+            // Update Follower
+            var twitterUserId = twitterUser.Id;
+            if(!follower.Followings.Contains(twitterUserId))
+                follower.Followings.Add(twitterUserId);
+
+            if(!follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
+                follower.FollowingsSyncStatus.Add(twitterUserId, -1);
+            
+            // Save Follower
+            await _followerDal.UpdateFollowerAsync(follower);
+        }
+    }
+}

src/BirdsiteLive.Domain/BusinessUseCases/ProcessUnfollowUser.cs

@@ -0,0 +1,45 @@
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+
+namespace BirdsiteLive.Domain.BusinessUseCases
+{
+    public interface IProcessUndoFollowUser
+    {
+        Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername);
+    }
+
+    public class ProcessUndoFollowUser : IProcessUndoFollowUser
+    {
+        private readonly IFollowersDal _followerDal;
+        private readonly ITwitterUserDal _twitterUserDal;
+
+        #region Ctor
+        public ProcessUndoFollowUser(IFollowersDal followerDal, ITwitterUserDal twitterUserDal)
+        {
+            _followerDal = followerDal;
+            _twitterUserDal = twitterUserDal;
+        }
+        #endregion
+
+        public async Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername)
+        {
+            // Get Follower and Twitter Users
+            var follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain);
+            if (follower == null) return;
+
+            var twitterUser = await _twitterUserDal.GetTwitterUserAsync(twitterUsername);
+            if (twitterUser == null) return;
+
+            // Update Follower
+            var twitterUserId = twitterUser.Id;
+            if (follower.Followings.Contains(twitterUserId))
+                follower.Followings.Remove(twitterUserId);
+
+            if (follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
+                follower.FollowingsSyncStatus.Remove(twitterUserId);
+
+            // Save Follower
+            await _followerDal.UpdateFollowerAsync(follower);
+        }
+    }
+}

src/BirdsiteLive.Domain/CryptoService.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Security.Cryptography;
 using System.Text;
 using BirdsiteLive.Domain.Factories;
 
@@ -7,7 +8,8 @@ namespace BirdsiteLive.Domain
     public interface ICryptoService
     {
         string GetUserPem(string id);
-        string SignAndGetSignatureHeader(DateTime date, string actor, string host, string inbox = null);
+        string SignAndGetSignatureHeader(DateTime date, string actor, string host, string digest, string inbox);
+        string ComputeSha256Hash(string data);
     }
 
     public class CryptoService : ICryptoService
@@ -33,7 +35,7 @@ namespace BirdsiteLive.Domain
         /// <param name="actor">in the form of https://domain.io/actor</param>
         /// <param name="host">in the form of domain.io</param>
         /// <returns></returns>
-        public string SignAndGetSignatureHeader(DateTime date, string actor, string targethost, string inbox = null)
+        public string SignAndGetSignatureHeader(DateTime date, string actor, string targethost, string digest, string inbox)
         {
             var usedInbox = "/inbox";
             if (!string.IsNullOrWhiteSpace(inbox))
@@ -41,13 +43,24 @@ namespace BirdsiteLive.Domain
 
             var httpDate = date.ToString("r");
 
-            var signedString = $"(request-target): post {usedInbox}\nhost: {targethost}\ndate: {httpDate}";
+            var signedString = $"(request-target): post {usedInbox}\nhost: {targethost}\ndate: {httpDate}\ndigest: SHA-256={digest}";
             var signedStringBytes = Encoding.UTF8.GetBytes(signedString);
             var signature = _magicKeyFactory.GetMagicKey().Sign(signedStringBytes);
             var sig64 = Convert.ToBase64String(signature);
 
-            var header = "keyId=\"" + actor + "\",headers=\"(request-target) host date\",signature=\"" + sig64 + "\"";
+            var header = "keyId=\"" + actor + "\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest\",signature=\"" + sig64 + "\"";
             return header;
         }
+
+        public string ComputeSha256Hash(string data)
+        {
+            // Create a SHA256   
+            using (SHA256 sha256Hash = SHA256.Create())
+            {
+                // ComputeHash - returns byte array  
+                byte[] bytes = sha256Hash.ComputeHash(Encoding.UTF8.GetBytes(data));
+                return Convert.ToBase64String(bytes);
+            }
+        }
     }
 }

src/BirdsiteLive.Domain/StatusService.cs

@@ -0,0 +1,90 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using BirdsiteLive.ActivityPub;
+using BirdsiteLive.ActivityPub.Models;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.Domain.Tools;
+using BirdsiteLive.Twitter.Models;
+using Tweetinvi.Models;
+using Tweetinvi.Models.Entities;
+
+namespace BirdsiteLive.Domain
+{
+    public interface IStatusService
+    {
+        Note GetStatus(string username, ExtractedTweet tweet);
+    }
+
+    public class StatusService : IStatusService
+    {
+        private readonly InstanceSettings _instanceSettings;
+        private readonly IStatusExtractor _statusExtractor;
+
+        #region Ctor
+        public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor)
+        {
+            _instanceSettings = instanceSettings;
+            _statusExtractor = statusExtractor;
+        }
+        #endregion
+
+        public Note GetStatus(string username, ExtractedTweet tweet)
+        {
+            var actorUrl = $"https://{_instanceSettings.Domain}/users/{username}";
+            var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{tweet.Id}";
+            var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{tweet.Id}";
+
+            var to = $"{actorUrl}/followers";
+            var apPublic = "https://www.w3.org/ns/activitystreams#Public";
+
+            var extractedTags = _statusExtractor.ExtractTags(tweet.MessageContent);
+
+            string inReplyTo = null;
+            if (tweet.InReplyToStatusId != default)
+                inReplyTo = $"https://{_instanceSettings.Domain}/users/{tweet.InReplyToAccount}/statuses/{tweet.InReplyToStatusId}";
+
+            var note = new Note
+            {
+                //id = $"{noteId}/activity",
+                id = $"{noteId}",
+
+                published = tweet.CreatedAt.ToString("s") + "Z",
+                url = noteUrl,
+                attributedTo = actorUrl,
+
+                inReplyTo = inReplyTo,
+                //to = new [] {to},
+                //cc = new [] { apPublic },
+
+                to = new[] { to },
+                //cc = new[] { apPublic },
+                cc = new string[0],
+
+                sensitive = false,
+                content = $"<p>{extractedTags.content}</p>",
+                attachment = Convert(tweet.Media),
+                tag = extractedTags.tags
+            };
+          
+
+            return note;
+        }
+
+        private Attachment[] Convert(ExtractedMedia[] media)
+        {
+            if(media == null) return new Attachment[0];
+            return media.Select(x =>
+            {
+                return new Attachment
+                {
+                    type = "Document",
+                    url = x.Url,
+                    mediaType = x.MediaType
+                };
+            }).ToArray();
+        }
+    }
+}

src/BirdsiteLive.Domain/Tools/StatusExtractor.cs

@@ -0,0 +1,130 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+using BirdsiteLive.ActivityPub.Models;
+using BirdsiteLive.Common.Settings;
+
+namespace BirdsiteLive.Domain.Tools
+{
+    public interface IStatusExtractor
+    {
+        (string content, Tag[] tags) ExtractTags(string messageContent);
+    }
+
+    public class StatusExtractor : IStatusExtractor
+    {
+        private readonly Regex _hastagRegex = new Regex(@"\W(\#[a-zA-Z0-9_ー]+\b)(?!;)");
+        //private readonly Regex _hastagRegex = new Regex(@"#\w+");
+        //private readonly Regex _hastagRegex = new Regex(@"(?<=[\s>]|^)#(\w*[a-zA-Z0-9_ー]+\w*)\b(?!;)");
+        //private readonly Regex _hastagRegex = new Regex(@"(?<=[\s>]|^)#(\w*[a-zA-Z0-9_ー]+)\b(?!;)");
+
+        private readonly Regex _mentionRegex = new Regex(@"\W(\@[a-zA-Z0-9_ー]+\b)(?!;)");
+        //private readonly Regex _mentionRegex = new Regex(@"@\w+");
+        //private readonly Regex _mentionRegex = new Regex(@"(?<=[\s>]|^)@(\w*[a-zA-Z0-9_ー]+\w*)\b(?!;)");
+        //private readonly Regex _mentionRegex = new Regex(@"(?<=[\s>]|^)@(\w*[a-zA-Z0-9_ー]+)\b(?!;)");
+
+        private readonly Regex _urlRegex = new Regex(@"((http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&amp;:/~\+#]*[\w\-\@?^=%&amp;/~\+#])?)");
+        private readonly InstanceSettings _instanceSettings;
+
+        #region Ctor
+        public StatusExtractor(InstanceSettings instanceSettings)
+        {
+            _instanceSettings = instanceSettings;
+        }
+        #endregion
+
+        public (string content, Tag[] tags) ExtractTags(string messageContent)
+        {
+            var tags = new List<Tag>();
+            messageContent = $" {messageContent} ";
+
+            // Replace return lines
+            messageContent = Regex.Replace(messageContent, @"\r\n\r\n?|\n\n", "</p><p> ");
+            messageContent = Regex.Replace(messageContent, @"\r\n?|\n", "<br/> ");
+
+            // Extract Urls
+            var urlMatch = _urlRegex.Matches(messageContent);
+            foreach (Match m in urlMatch)
+            {
+                var url = m.ToString().Replace("\n", string.Empty).Trim();
+
+                var protocol = "https://";
+                if (url.StartsWith("http://")) protocol = "http://";
+                else if (url.StartsWith("ftp://")) protocol = "ftp://";
+
+                var truncatedUrl = url.Replace(protocol, string.Empty);
+
+                if (truncatedUrl.StartsWith("www."))
+                {
+                    protocol += "www.";
+                    truncatedUrl = truncatedUrl.Replace("www.", string.Empty);
+                }
+
+                var firstPart = truncatedUrl;
+                var secondPart = string.Empty;
+
+                if (truncatedUrl.Length > 30)
+                {
+                    firstPart = truncatedUrl.Substring(0, 30);
+                    secondPart = truncatedUrl.Substring(30);
+                }
+
+                messageContent = Regex.Replace(messageContent, m.ToString(),
+                    $@" <a href=""{url}"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">{protocol}</span><span class=""ellipsis"">{firstPart}</span><span class=""invisible"">{secondPart}</span></a>");
+            }
+
+            // Extract Hashtags
+            var hashtagMatch = OrderByLength(_hastagRegex.Matches(messageContent));
+            foreach (Match m in hashtagMatch)
+            {
+                var tag = m.ToString().Replace("#", string.Empty).Replace("\n", string.Empty).Trim();
+                var url = $"https://{_instanceSettings.Domain}/tags/{tag}";
+
+                tags.Add(new Tag
+                {
+                    name = $"#{tag}",
+                    href = url,
+                    type = "Hashtag"
+                });
+
+                messageContent = Regex.Replace(messageContent, m.ToString(),
+                    $@" <a href=""{url}"" class=""mention hashtag"" rel=""tag"">#<span>{tag}</span></a>");
+            }
+
+            // Extract Mentions
+            var mentionMatch = OrderByLength(_mentionRegex.Matches(messageContent));
+            foreach (Match m in mentionMatch)
+            {
+                var mention = m.ToString().Replace("@", string.Empty).Replace("\n", string.Empty).Trim();
+                var url = $"https://{_instanceSettings.Domain}/users/{mention}";
+                var name = $"@{mention}@{_instanceSettings.Domain}";
+
+                tags.Add(new Tag
+                {
+                    name = name,
+                    href = url,
+                    type = "Mention"
+                });
+
+                messageContent = Regex.Replace(messageContent, m.ToString(),
+                    $@" <span class=""h-card""><a href=""https://{_instanceSettings.Domain}/@{mention}"" class=""u-url mention"">@<span>{mention}</span></a></span>");
+            }
+
+            // Clean up return lines
+            messageContent = Regex.Replace(messageContent, @"<p> ", "<p>");
+            messageContent = Regex.Replace(messageContent, @"<br/> ", "<br/>");
+
+            return (messageContent.Trim(), tags.ToArray());
+        }
+
+        private IEnumerable<Match> OrderByLength(MatchCollection matches)
+        {
+            var result = new List<Match>();
+
+            foreach (Match m in matches) result.Add(m);
+            result = result.OrderByDescending(x => x.Length).ToList();
+
+            return result;
+        }
+    }
+}

src/BirdsiteLive.Domain/UserService.cs

@@ -8,6 +8,7 @@ using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Cryptography;
+using BirdsiteLive.Domain.BusinessUseCases;
 using BirdsiteLive.Twitter.Models;
 using Tweetinvi.Core.Exceptions;
 using Tweetinvi.Models;
@@ -17,22 +18,28 @@ namespace BirdsiteLive.Domain
     public interface IUserService
     {
         Actor GetUser(TwitterUser twitterUser);
-        Task<bool> FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityFollow activity);
-        Note GetStatus(TwitterUser user, ITweet tweet);
+        Task<bool> FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityFollow activity, string body);
+        Task<bool> UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityUndoFollow activity, string body);
     }
 
     public class UserService : IUserService
     {
+        private readonly IProcessFollowUser _processFollowUser;
+        private readonly IProcessUndoFollowUser _processUndoFollowUser;
+
+        private readonly InstanceSettings _instanceSettings;
         private readonly ICryptoService _cryptoService;
         private readonly IActivityPubService _activityPubService;
-        private readonly string _host;
 
         #region Ctor
-        public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService)
+        public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser)
         {
+            _instanceSettings = instanceSettings;
             _cryptoService = cryptoService;
             _activityPubService = activityPubService;
-            _host = $"https://{instanceSettings.Domain.Replace("https://",string.Empty).Replace("http://", string.Empty).TrimEnd('/')}";
+            _processFollowUser = processFollowUser;
+            _processUndoFollowUser = processUndoFollowUser;
+            //_host = $"https://{instanceSettings.Domain.Replace("https://",string.Empty).Replace("http://", string.Empty).TrimEnd('/')}";
         }
         #endregion
 
@@ -40,17 +47,18 @@ namespace BirdsiteLive.Domain
         {
             var user = new Actor
             {
-                id = $"{_host}/users/{twitterUser.Acct}",
-                type = "Person",
+                id = $"https://{_instanceSettings.Domain}/users/{twitterUser.Acct}",
+                type = "Service", //Person Service
+                followers = $"https://{_instanceSettings.Domain}/users/{twitterUser.Acct}/followers",
                 preferredUsername = twitterUser.Acct,
                 name = twitterUser.Name,
-                inbox = $"{_host}/users/{twitterUser.Acct}/inbox",
+                inbox = $"https://{_instanceSettings.Domain}/users/{twitterUser.Acct}/inbox",
                 summary = twitterUser.Description,
-                url = $"{_host}/@{twitterUser.Acct}",
+                url = $"https://{_instanceSettings.Domain}/@{twitterUser.Acct}",
                 publicKey = new PublicKey()
                 {
-                    id = $"{_host}/users/{twitterUser.Acct}#main-key",
-                    owner = $"{_host}/users/{twitterUser.Acct}",
+                    id = $"https://{_instanceSettings.Domain}/users/{twitterUser.Acct}#main-key",
+                    owner = $"https://{_instanceSettings.Domain}/users/{twitterUser.Acct}",
                     publicKeyPem = _cryptoService.GetUserPem(twitterUser.Acct)
                 },
                 icon = new Image
@@ -62,60 +70,87 @@ namespace BirdsiteLive.Domain
                 {
                     mediaType = "image/jpeg",
                     url = twitterUser.ProfileBannerURL
+                },
+                endpoints = new EndPoints
+                {
+                    sharedInbox = $"https://{_instanceSettings.Domain}/inbox"
                 }
             };
             return user;
         }
 
-        public Note GetStatus(TwitterUser user, ITweet tweet)
+        public async Task<bool> FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityFollow activity, string body)
         {
-            var actor = GetUser(user);
-
-            var actorUrl = $"{_host}/users/{user.Acct}";
-            var noteId = $"{_host}/users/{user.Acct}/statuses/{tweet.Id}";
-            var noteUrl = $"{_host}/@{user.Acct}/{tweet.Id}";
+            // Validate
+            var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
+            if (!sigValidation.SignatureIsValidated) return false;
 
-            var to = $"{actor}/followers";
-            var apPublic = "https://www.w3.org/ns/activitystreams#Public";
+            // Save Follow in DB
+            var followerUserName = sigValidation.User.preferredUsername.ToLowerInvariant();
+            var followerHost = sigValidation.User.url.Replace("https://", string.Empty).Split('/').First();
+            var followerInbox = sigValidation.User.inbox;
+            var followerSharedInbox = sigValidation.User?.endpoints?.sharedInbox;
+            var twitterUser = activity.apObject.Split('/').Last().Replace("@", string.Empty);
+
+            // Make sure to only keep routes
+            followerInbox = OnlyKeepRoute(followerInbox, followerHost);
+            followerSharedInbox = OnlyKeepRoute(followerSharedInbox, followerHost);
+            
+            // Execute
+            await _processFollowUser.ExecuteAsync(followerUserName, followerHost, twitterUser, followerInbox, followerSharedInbox);
 
-            var note = new Note
+            // Send Accept Activity
+            var acceptFollow = new ActivityAcceptFollow()
             {
-                id = $"{noteId}/activity",
-                
-                published = tweet.CreatedAt.ToString("s") + "Z",
-                url = noteUrl,
-                attributedTo = actorUrl,
-
-                //to = new [] {to},
-                //cc = new [] { apPublic },
-
-                to = new[] { apPublic },
-                cc = new[] { to },
-
-                sensitive = false,
-                content = $"<p>{tweet.Text}</p>",
-                attachment = new string[0],
-                tag = new string[0]
+                context = "https://www.w3.org/ns/activitystreams",
+                id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
+                type = "Accept",
+                actor = activity.apObject,
+                apObject = new ActivityFollow()
+                {
+                    id = activity.id,
+                    type = activity.type,
+                    actor = activity.actor,
+                    apObject = activity.apObject
+                }
             };
-            return note;
+            var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
+            return result == HttpStatusCode.Accepted;
+        }
+
+        private string OnlyKeepRoute(string inbox, string host)
+        {
+            if (string.IsNullOrWhiteSpace(inbox)) 
+                return null;
+
+            if (inbox.Contains(host))
+                inbox = inbox.Split(new[] { host }, StringSplitOptions.RemoveEmptyEntries).Last();
+
+            return inbox;
         }
 
-        public async Task<bool> FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string>  requestHeaders, ActivityFollow activity)
+        public async Task<bool> UndoFollowRequestedAsync(string signature, string method, string path, string queryString,
+            Dictionary<string, string> requestHeaders, ActivityUndoFollow activity, string body)
         {
             // Validate
-            if (!await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders)) return false;
+            var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
+            if (!sigValidation.SignatureIsValidated) return false;
 
             // Save Follow in DB
-            
-            // Send Accept Activity 
-            var targetHost = activity.actor.Replace("https://", string.Empty).Split('/').First();
-            var acceptFollow = new ActivityAcceptFollow()
+            var followerUserName = sigValidation.User.name.ToLowerInvariant();
+            var followerHost = sigValidation.User.url.Replace("https://", string.Empty).Split('/').First();
+            //var followerInbox = sigValidation.User.inbox;
+            var twitterUser = activity.apObject.apObject.Split('/').Last().Replace("@", string.Empty);
+            await _processUndoFollowUser.ExecuteAsync(followerUserName, followerHost, twitterUser);
+
+            // Send Accept Activity
+            var acceptFollow = new ActivityAcceptUndoFollow()
             {
                 context = "https://www.w3.org/ns/activitystreams",
-                id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
+                id = $"{activity.apObject.apObject}#accepts/undofollows/{Guid.NewGuid()}",
                 type = "Accept",
-                actor = activity.apObject,
-                apObject = new ActivityFollow()
+                actor = activity.apObject.apObject,
+                apObject = new ActivityUndoFollow()
                 {
                     id = activity.id,
                     type = activity.type,
@@ -123,12 +158,26 @@ namespace BirdsiteLive.Domain
                     apObject = activity.apObject
                 }
             };
-            var result = await _activityPubService.PostDataAsync(acceptFollow, targetHost, activity.apObject);
+            var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject.apObject);
             return result == HttpStatusCode.Accepted;
         }
-        
-        private async Task<bool> ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary<string, string> requestHeaders)
+
+        private async Task<SignatureValidationResult> ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary<string, string> requestHeaders, string body)
         {
+            //Check Date Validity
+            var date = requestHeaders["date"];
+            var d = DateTime.Parse(date).ToUniversalTime();
+            var now = DateTime.UtcNow;
+            var delta = Math.Abs((d - now).TotalSeconds);
+            if (delta > 30) return new SignatureValidationResult { SignatureIsValidated = false };
+            
+            //Check Digest
+            var digest = requestHeaders["digest"];
+            var digestHash = digest.Split(new [] {"SHA-256="},StringSplitOptions.RemoveEmptyEntries).LastOrDefault();
+            var calculatedDigestHash = _cryptoService.ComputeSha256Hash(body);
+            if (digestHash != calculatedDigestHash) return new SignatureValidationResult { SignatureIsValidated = false };
+
+            //Check Signature
             var signatures = rawSig.Split(',');
             var signature_header = new Dictionary<string, string>();
             foreach (var signature in signatures)
@@ -184,7 +233,17 @@ namespace BirdsiteLive.Domain
 
             var result = signKey.VerifyData(Encoding.UTF8.GetBytes(toSign.ToString()), sig, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
 
-            return result;
+            return new SignatureValidationResult()
+            {
+                SignatureIsValidated = result,
+                User = remoteUser
+            };
         }
     }
+
+    public class SignatureValidationResult 
+    {
+        public bool SignatureIsValidated { get; set; }
+        public Actor User { get; set; }
+    }
 }

src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <LangVersion>latest</LangVersion>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="1.1.1" />
+    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.11.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\BirdsiteLive.Domain\BirdsiteLive.Domain.csproj" />
+    <ProjectReference Include="..\BirdsiteLive.Twitter\BirdsiteLive.Twitter.csproj" />
+    <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
+  </ItemGroup>
+
+</Project>

src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs

@@ -0,0 +1,13 @@
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts
+{
+    public interface IRetrieveFollowersProcessor
+    {
+        Task<IEnumerable<UserWithTweetsToSync>> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct);
+        //IAsyncEnumerable<UserWithTweetsToSync> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct);
+    }
+}

src/BirdsiteLive.Pipeline/Contracts/IRetrieveTweetsProcessor.cs

@@ -0,0 +1,12 @@
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts
+{
+    public interface IRetrieveTweetsProcessor
+    {
+        Task<UserWithTweetsToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct);
+    }
+}

src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs

@@ -0,0 +1,12 @@
+using System.Threading;
+using System.Threading.Tasks;
+using System.Threading.Tasks.Dataflow;
+using BirdsiteLive.DAL.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts
+{
+    public interface IRetrieveTwitterUsersProcessor
+    {
+        Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct);
+    }
+}

src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs

@@ -0,0 +1,11 @@
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts
+{
+    public interface ISaveProgressionProcessor
+    {
+        Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct);
+    }
+}

src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs

@@ -0,0 +1,11 @@
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts
+{
+    public interface ISendTweetsToFollowersProcessor
+    {
+        Task<UserWithTweetsToSync> ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct);
+    }
+}

src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs

@@ -0,0 +1,13 @@
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Twitter.Models;
+using Tweetinvi.Models;
+
+namespace BirdsiteLive.Pipeline.Models
+{
+    public class UserWithTweetsToSync
+    {
+        public SyncTwitterUser User { get; set; }
+        public ExtractedTweet[] Tweets { get; set; }
+        public Follower[] Followers { get; set; }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs

@@ -0,0 +1,33 @@
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Processors
+{
+    public class RetrieveFollowersProcessor : IRetrieveFollowersProcessor
+    {
+        private readonly IFollowersDal _followersDal;
+
+        #region Ctor
+        public RetrieveFollowersProcessor(IFollowersDal followersDal)
+        {
+            _followersDal = followersDal;
+        }
+        #endregion
+
+        public async Task<IEnumerable<UserWithTweetsToSync>> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct)
+        {
+            //TODO multithread this
+            foreach (var user in userWithTweetsToSyncs)
+            {
+                var followers = await _followersDal.GetFollowersAsync(user.User.Id);
+                user.Followers = followers;
+            }
+
+            return userWithTweetsToSyncs;
+        }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs

@@ -0,0 +1,66 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Models;
+using BirdsiteLive.Twitter;
+using BirdsiteLive.Twitter.Models;
+using Tweetinvi.Models;
+
+namespace BirdsiteLive.Pipeline.Processors
+{
+    public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor
+    {
+        private readonly ITwitterService _twitterService;
+        private readonly ITwitterUserDal _twitterUserDal;
+
+        #region Ctor
+        public RetrieveTweetsProcessor(ITwitterService twitterService, ITwitterUserDal twitterUserDal)
+        {
+            _twitterService = twitterService;
+            _twitterUserDal = twitterUserDal;
+        }
+        #endregion
+
+        public async Task<UserWithTweetsToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct)
+        {
+            var usersWtTweets = new List<UserWithTweetsToSync>();
+
+            //TODO multithread this
+            foreach (var user in syncTwitterUsers)
+            {
+                var tweets = RetrieveNewTweets(user);
+                if (tweets.Length > 0 && user.LastTweetPostedId != -1)
+                {
+                    var userWtTweets = new UserWithTweetsToSync
+                    {
+                        User = user,
+                        Tweets = tweets
+                    };
+                    usersWtTweets.Add(userWtTweets);
+                }
+                else if (tweets.Length > 0 && user.LastTweetPostedId == -1)
+                {
+                    var tweetId = tweets.Last().Id;
+                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId);
+                }
+            }
+
+            return usersWtTweets.ToArray();
+        }
+
+        private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
+        {
+            ExtractedTweet[] tweets;
+            if (user.LastTweetPostedId == -1)
+                tweets = _twitterService.GetTimeline(user.Acct, 1);
+            else
+                tweets = _twitterService.GetTimeline(user.Acct, 200, user.LastTweetSynchronizedForAllFollowersId);
+
+            return tweets;
+        }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs

@@ -0,0 +1,45 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Threading.Tasks.Dataflow;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Pipeline.Contracts;
+
+namespace BirdsiteLive.Pipeline.Processors
+{
+    public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
+    {
+        private readonly ITwitterUserDal _twitterUserDal;
+        private const int SyncPeriod = 15; //in minutes
+
+        #region Ctor
+        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal)
+        {
+            _twitterUserDal = twitterUserDal;
+        }
+        #endregion
+
+        public async Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct)
+        {
+            for (;;)
+            {
+                ct.ThrowIfCancellationRequested();
+
+                try
+                {
+                    var users = await _twitterUserDal.GetAllTwitterUsersAsync();
+
+                    if(users.Length > 0)
+                        await twitterUsersBufferBlock.SendAsync(users, ct);
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e);
+                }
+
+                await Task.Delay(SyncPeriod * 1000 * 60, ct);
+            }
+        }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs

@@ -0,0 +1,29 @@
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Processors
+{
+    public class SaveProgressionProcessor : ISaveProgressionProcessor
+    {
+        private readonly ITwitterUserDal _twitterUserDal;
+        
+        #region Ctor
+        public SaveProgressionProcessor(ITwitterUserDal twitterUserDal)
+        {
+            _twitterUserDal = twitterUserDal;
+        }
+        #endregion
+
+        public async Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct)
+        {
+            var userId = userWithTweetsToSync.User.Id;
+            var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
+            var minimumSync = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).Min();
+            await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync);
+        }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs

@@ -0,0 +1,86 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Xml;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Domain;
+using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Models;
+using BirdsiteLive.Pipeline.Processors.SubTasks;
+using BirdsiteLive.Twitter;
+using BirdsiteLive.Twitter.Models;
+using Tweetinvi.Models;
+
+namespace BirdsiteLive.Pipeline.Processors
+{
+    public class SendTweetsToFollowersProcessor : ISendTweetsToFollowersProcessor
+    {
+        private readonly ISendTweetsToInboxTask _sendTweetsToInboxTask;
+        private readonly ISendTweetsToSharedInboxTask _sendTweetsToSharedInbox;
+
+        #region Ctor
+        public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox)
+        {
+            _sendTweetsToInboxTask = sendTweetsToInboxTask;
+            _sendTweetsToSharedInbox = sendTweetsToSharedInbox;
+        }
+        #endregion
+
+        public async Task<UserWithTweetsToSync> ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct)
+        {
+            var user = userWithTweetsToSync.User;
+
+            // Process Shared Inbox
+            var followersWtSharedInbox = userWithTweetsToSync.Followers
+                .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
+                .ToList();
+            await ProcessFollowersWithSharedInbox(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
+
+            // Process Inbox
+            var followerWtInbox = userWithTweetsToSync.Followers
+                .Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
+                .ToList();
+            await ProcessFollowersWithInbox(userWithTweetsToSync.Tweets, followerWtInbox, user);
+
+            return userWithTweetsToSync;
+        }
+
+        private async Task ProcessFollowersWithSharedInbox(ExtractedTweet[] tweets, List<Follower> followers, SyncTwitterUser user)
+        {
+            var followersPerInstances = followers.GroupBy(x => x.Host);
+
+            foreach (var followersPerInstance in followersPerInstances)
+            {
+                try
+                {
+                    await _sendTweetsToSharedInbox.ExecuteAsync(tweets, user, followersPerInstance.Key, followersPerInstance.ToArray());
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e);
+                    //TODO handle error
+                }
+            }
+        }
+        
+        private async Task ProcessFollowersWithInbox(ExtractedTweet[] tweets, List<Follower> followerWtInbox, SyncTwitterUser user)
+        {
+            foreach (var follower in followerWtInbox)
+            {
+                try
+                {
+                    await _sendTweetsToInboxTask.ExecuteAsync(tweets, follower, user);
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e);
+                    //TODO handle error
+                }
+            }
+        }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs

@@ -0,0 +1,68 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Domain;
+using BirdsiteLive.Twitter.Models;
+
+namespace BirdsiteLive.Pipeline.Processors.SubTasks
+{
+    public interface ISendTweetsToInboxTask
+    {
+        Task ExecuteAsync(IEnumerable<ExtractedTweet> tweets, Follower follower, SyncTwitterUser user);
+    }
+
+    public class SendTweetsToInboxTask : ISendTweetsToInboxTask
+    {
+        private readonly IActivityPubService _activityPubService;
+        private readonly IStatusService _statusService;
+        private readonly IFollowersDal _followersDal;
+
+        #region Ctor
+        public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal)
+        {
+            _activityPubService = activityPubService;
+            _statusService = statusService;
+            _followersDal = followersDal;
+        }
+        #endregion
+
+        public async Task ExecuteAsync(IEnumerable<ExtractedTweet> tweets, Follower follower, SyncTwitterUser user)
+        {
+            var userId = user.Id;
+            var fromStatusId = follower.FollowingsSyncStatus[userId];
+            var tweetsToSend = tweets
+                .Where(x => x.Id > fromStatusId)
+                .OrderBy(x => x.Id)
+                .ToList();
+
+            var inbox = follower.InboxRoute;
+
+            var syncStatus = fromStatusId;
+            try
+            {
+                foreach (var tweet in tweetsToSend)
+                {
+                    var note = _statusService.GetStatus(user.Acct, tweet);
+                    var result = await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
+
+                    if (result == HttpStatusCode.Accepted)
+                        syncStatus = tweet.Id;
+                    else
+                        throw new Exception("Posting new note activity failed");
+                }
+            }
+            finally
+            {
+                if (syncStatus != fromStatusId)
+                {
+                    follower.FollowingsSyncStatus[userId] = syncStatus;
+                    await _followersDal.UpdateFollowerAsync(follower);
+                }
+            }
+        }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs

@@ -0,0 +1,73 @@
+using System;
+using System.Linq;
+using System.Net;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Domain;
+using BirdsiteLive.Twitter.Models;
+
+namespace BirdsiteLive.Pipeline.Processors.SubTasks
+{
+    public interface ISendTweetsToSharedInboxTask
+    {
+        Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance);
+    }
+
+    public class SendTweetsToSharedInboxTask : ISendTweetsToSharedInboxTask
+    {
+        private readonly IStatusService _statusService;
+        private readonly IActivityPubService _activityPubService;
+        private readonly IFollowersDal _followersDal;
+        
+        #region Ctor
+        public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal)
+        {
+            _activityPubService = activityPubService;
+            _statusService = statusService;
+            _followersDal = followersDal;
+        }
+        #endregion
+
+        public async Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance)
+        {